From 465d386da5237f011cb408f1a3d4b7d0c6d8ffae Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 16 Aug 2026 19:45:40 +0000 Subject: [PATCH 01/46] Repair AI usage exporter poller health --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/scripts/ai_usage_exporter.py | 273 +++++++++++++++--- .../tests/test_hermes_ai_usage_exporter.py | 218 ++++++++++++++ 3 files changed, 451 insertions(+), 42 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 845eb6f9..526c5c55 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6" + ai.bstein.dev/config-rev: "20260816-ai-usage-poller-v1" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index 178c2c54..16be50a6 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -48,6 +48,9 @@ METRIC_HELP = { "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.", "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.", } +CODEX_CLEANUP_TIMEOUT_SECONDS = 5 +POLL_STARTUP_GRACE_SECONDS = 45 +POLL_PROGRESS_BUDGET_SECONDS = 60 @dataclass(frozen=True) @@ -228,6 +231,63 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: return samples +def _close_stream(stream: Any) -> bool: + """Close a subprocess pipe without allowing cleanup errors to escape.""" + if stream is None: + return True + try: + stream.close() + except Exception: + return False + return True + + +def _cleanup_codex_process( + process: subprocess.Popen[str], selector: selectors.BaseSelector | None +) -> bool: + """Stop and reap a Codex child within two fixed waits, then close its pipes.""" + cleanup_ok = True + if selector is not None: + try: + selector.close() + except Exception: + cleanup_ok = False + + # Closing the request pipe first also gives a responsive app-server an EOF. + cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok + try: + process.terminate() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + + needs_kill = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + needs_kill = True + except Exception: + cleanup_ok = False + needs_kill = True + + if needs_kill: + try: + process.kill() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except Exception: + cleanup_ok = False + + cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok + cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok + return cleanup_ok + + def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: """Read Codex account quota and usage through its structured app-server protocol.""" process = subprocess.Popen( @@ -256,6 +316,9 @@ def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: {"id": 2, "method": "account/rateLimits/read", "params": None}, {"id": 3, "method": "account/usage/read", "params": None}, ) + selector: selectors.BaseSelector | None = None + rate_result: dict[str, Any] + usage_result: dict[str, Any] try: if process.stdin is None or process.stdout is None: raise RuntimeError("Codex app-server pipes are unavailable") @@ -278,14 +341,13 @@ def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: response = responses.get(response_id) if not response or "error" in response or not isinstance(response.get("result"), dict): raise RuntimeError(f"Codex app-server response {response_id} failed") - return responses[2]["result"], responses[3]["result"] + rate_result = responses[2]["result"] + usage_result = responses[3]["result"] finally: - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) + cleanup_ok = _cleanup_codex_process(process, selector) + if not cleanup_ok: + raise RuntimeError("Codex app-server cleanup failed") + return rate_result, usage_result def query_claude() -> dict[str, Any]: @@ -322,6 +384,22 @@ class Collector: self._lock = threading.Lock() self._providers = {name: ProviderState() for name in ("openai", "anthropic")} + def record_failure( + self, + provider: str, + *, + started: float | None = None, + monotonic_started: float | None = None, + ) -> None: + """Record a failed attempt while retaining the provider's last good samples.""" + attempted_at = time.time() if started is None else started + duration_started = time.monotonic() if monotonic_started is None else monotonic_started + with self._lock: + state = self._providers[provider] + state.last_attempt = attempted_at + state.duration = max(0, time.monotonic() - duration_started) + state.fetch_success = False + def refresh_provider(self, provider: str) -> None: """Refresh one provider while retaining the last good values on failure.""" started = time.time() @@ -333,19 +411,21 @@ class Collector: samples = parse_claude_payload(query_claude()) else: raise ValueError("unknown provider") - success = True - except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + except Exception as error: print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True) - samples = [] - success = False + self.record_failure( + provider, + started=started, + monotonic_started=monotonic_started, + ) + return with self._lock: state = self._providers[provider] state.last_attempt = started state.duration = time.monotonic() - monotonic_started - state.fetch_success = success - if success: - state.samples = samples - state.last_success = time.time() + state.fetch_success = True + state.samples = samples + state.last_success = time.time() def render(self) -> bytes: """Render the current provider states in Prometheus text format.""" @@ -381,38 +461,149 @@ class Collector: return ("\n".join(lines) + "\n").encode("utf-8") +class PollingEngine: + """Run isolated provider polls and track bounded forward progress.""" + + def __init__( + self, + collector: Collector, + *, + interval: float, + startup_grace: float = POLL_STARTUP_GRACE_SECONDS, + progress_timeout: float | None = None, + clock: Any = time.monotonic, + ) -> None: + self.collector = collector + self.interval = interval + self.startup_grace = startup_grace + self.progress_timeout = ( + interval + POLL_PROGRESS_BUDGET_SECONDS + if progress_timeout is None + else progress_timeout + ) + self._clock = clock + self._started_at = clock() + self._last_progress: float | None = None + self._progress_lock = threading.Lock() + self._thread: threading.Thread | None = None + + def _mark_progress(self) -> None: + """Record completion of one bounded provider attempt.""" + with self._progress_lock: + self._last_progress = self._clock() + + def poll_once(self) -> None: + """Refresh every provider even if another provider fails unexpectedly.""" + for provider in ("openai", "anthropic"): + started = time.time() + monotonic_started = time.monotonic() + try: + self.collector.refresh_provider(provider) + except Exception as error: + print( + f"{provider} quota collection isolated: {type(error).__name__}", + flush=True, + ) + try: + self.collector.record_failure( + provider, + started=started, + monotonic_started=monotonic_started, + ) + except Exception as record_error: + print( + f"{provider} quota failure accounting deferred: " + f"{type(record_error).__name__}", + flush=True, + ) + finally: + self._mark_progress() + + def run(self) -> None: + """Poll forever without allowing a cycle-level exception to stop the thread.""" + while True: + try: + self.poll_once() + except Exception as error: + print(f"quota polling cycle deferred: {type(error).__name__}", flush=True) + time.sleep(self.interval) + + def start(self) -> None: + """Start the daemon poller exactly once.""" + if self._thread is not None: + return + self._thread = threading.Thread( + target=self.run, + name="ai-usage-poller", + daemon=True, + ) + self._thread.start() + + def is_healthy(self) -> bool: + """Report thread liveness and progress, independent of provider success.""" + thread = self._thread + if thread is None or not thread.is_alive(): + return False + now = self._clock() + with self._progress_lock: + last_progress = self._last_progress + if last_progress is None: + return now - self._started_at <= self.startup_grace + return now - last_progress <= self.progress_timeout + + +def make_handler(collector: Collector, poller: PollingEngine) -> type[BaseHTTPRequestHandler]: + """Build an HTTP handler bound to one collector and polling engine.""" + + class Handler(BaseHTTPRequestHandler): + def _respond(self, status: int, payload: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: # noqa: N802 + try: + if self.path == "/metrics": + self._respond( + 200, + collector.render(), + "text/plain; version=0.0.4", + ) + elif self.path == "/healthz": + healthy = poller.is_healthy() + self._respond( + 200 if healthy else 503, + b"ok\n" if healthy else b"poller unhealthy\n", + "text/plain; charset=utf-8", + ) + else: + self.send_error(404) + except (BrokenPipeError, ConnectionResetError): + return + + def log_message(self, _format: str, *_args: object) -> None: + return + + return Handler + + +class Server(ThreadingHTTPServer): + """Threaded metrics server that does not retain disconnected clients.""" + + daemon_threads = True + + def main() -> int: """Poll quota APIs and serve only sanitized metrics and health endpoints.""" collector = Collector() interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300"))) - def polling_loop() -> None: - while True: - for provider in ("openai", "anthropic"): - collector.refresh_provider(provider) - time.sleep(interval) - - class Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - if self.path == "/metrics": - payload = collector.render() - self.send_response(200) - self.send_header("Content-Type", "text/plain; version=0.0.4") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - elif self.path == "/healthz": - self.send_response(200) - self.end_headers() - else: - self.send_error(404) - - def log_message(self, _format: str, *_args: object) -> None: - return - - threading.Thread(target=polling_loop, daemon=True).start() + poller = PollingEngine(collector, interval=interval) + poller.start() port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010")) - server = ThreadingHTTPServer(("0.0.0.0", port), Handler) + server = Server(("0.0.0.0", port), make_handler(collector, poller)) server.serve_forever() return 0 diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index 9b78b7f3..9fe54d72 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -4,9 +4,12 @@ from __future__ import annotations import importlib.util import json +import subprocess import sys from datetime import date +from io import BytesIO, StringIO from pathlib import Path +from types import SimpleNamespace ROOT = Path(__file__).resolve().parents[2] @@ -173,3 +176,218 @@ for response_id, result in responses.items(): "account/usage/read", ] assert all("/status" not in json.dumps(request) for request in requests) + + +def test_codex_cleanup_is_bounded_when_process_cannot_be_reaped(monkeypatch): + mod = load_module() + processes = [] + selectors = [] + + class StubbornProcess: + def __init__(self, *_args, **_kwargs): + self.stdin = StringIO() + self.stdout = StringIO( + "\n".join( + json.dumps({"id": response_id, "result": result}) + for response_id, result in ( + (1, {}), + (2, {"rateLimits": {}}), + (3, {"summary": {}, "dailyUsageBuckets": []}), + ) + ) + + "\n" + ) + self.terminate_calls = 0 + self.kill_calls = 0 + self.wait_timeouts = [] + processes.append(self) + + def terminate(self): + self.terminate_calls += 1 + + def kill(self): + self.kill_calls += 1 + + def wait(self, timeout): + self.wait_timeouts.append(timeout) + raise subprocess.TimeoutExpired("codex-mock", timeout) + + class TrackingSelector: + def __init__(self): + self.fileobj = None + self.closed = False + selectors.append(self) + + def register(self, fileobj, _events): + self.fileobj = fileobj + + def select(self, _timeout): + return [(SimpleNamespace(fileobj=self.fileobj), None)] + + def close(self): + self.closed = True + + monkeypatch.setattr(mod.subprocess, "Popen", StubbornProcess) + monkeypatch.setattr(mod.selectors, "DefaultSelector", TrackingSelector) + + try: + mod.query_codex(timeout=1) + except Exception as error: # The cleanup result must be sanitized and non-TimeoutExpired. + assert type(error) is RuntimeError + assert str(error) == "Codex app-server cleanup failed" + else: + raise AssertionError("unreaped Codex process was reported as successful") + + process = processes[0] + assert process.terminate_calls == 1 + assert process.kill_calls == 1 + assert process.wait_timeouts == [5, 5] + assert process.stdin.closed + assert process.stdout.closed + assert selectors[0].closed + + +def test_unexpected_provider_failure_is_sanitized_and_preserves_last_good(monkeypatch, capsys): + mod = load_module() + collector = mod.Collector() + previous = mod.Sample( + "atlas_ai_quota_used_percent", + {"provider": "openai", "limit": "codex", "window": "seven_day"}, + 42, + ) + collector._providers["openai"].samples = [previous] + secret = "provider-body-must-not-appear" + + class UnexpectedProviderFailure(Exception): + pass + + def fail(): + raise UnexpectedProviderFailure(secret) + + monkeypatch.setattr(mod, "query_codex", fail) + collector.refresh_provider("openai") + + output = capsys.readouterr().out + state = collector._providers["openai"] + assert state.samples == [previous] + assert state.fetch_success is False + assert state.last_attempt > 0 + assert "UnexpectedProviderFailure" in output + assert secret not in output + + +def test_polling_engine_isolates_providers_and_future_cycles(monkeypatch, capsys): + mod = load_module() + collector = mod.Collector() + calls = [] + secret = "cleanup-detail-must-not-appear" + + class UnexpectedCleanupFailure(Exception): + pass + + def refresh(provider): + calls.append(provider) + if provider == "openai" and calls.count("openai") == 1: + raise UnexpectedCleanupFailure(secret) + + monkeypatch.setattr(collector, "refresh_provider", refresh) + engine = mod.PollingEngine(collector, interval=60) + + engine.poll_once() + engine.poll_once() + + output = capsys.readouterr().out + assert calls == ["openai", "anthropic", "openai", "anthropic"] + assert collector._providers["openai"].fetch_success is False + assert collector._providers["openai"].last_attempt > 0 + assert "UnexpectedCleanupFailure" in output + assert secret not in output + + +def test_health_tracks_poller_liveness_and_progress_not_provider_success(): + mod = load_module() + now = [100.0] + collector = mod.Collector() + collector._providers["openai"].fetch_success = False + engine = mod.PollingEngine( + collector, + interval=60, + startup_grace=10, + progress_timeout=75, + clock=lambda: now[0], + ) + + class ThreadState: + alive = True + + def is_alive(self): + return self.alive + + thread = ThreadState() + engine._thread = thread + assert engine.is_healthy() + + now[0] = 111.0 + assert not engine.is_healthy() + + engine._mark_progress() + assert engine.is_healthy() + now[0] = 187.0 + assert not engine.is_healthy() + + thread.alive = False + now[0] = 100.0 + assert not engine.is_healthy() + + +def test_health_endpoint_uses_poller_state_and_ignores_provider_failure(): + mod = load_module() + collector = mod.Collector() + + class Poller: + healthy = False + + def is_healthy(self): + return self.healthy + + poller = Poller() + handler_class = mod.make_handler(collector, poller) + handler = object.__new__(handler_class) + handler.path = "/healthz" + statuses = [] + handler.send_response = statuses.append + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = BytesIO() + + handler.do_GET() + assert statuses == [503] + + poller.healthy = True + statuses.clear() + collector._providers["openai"].fetch_success = False + handler.do_GET() + assert statuses == [200] + + +def test_metrics_response_tolerates_disconnected_client(): + mod = load_module() + collector = mod.Collector() + + class HealthyPoller: + def is_healthy(self): + return True + + class Disconnected(BytesIO): + def write(self, _body): + raise BrokenPipeError + + handler_class = mod.make_handler(collector, HealthyPoller()) + handler = object.__new__(handler_class) + handler.path = "/metrics" + handler.send_response = lambda _status: None + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = Disconnected() + + handler.do_GET() From 8f005458282269ba5c07941814e4237f5d4cf3ac Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 10:14:17 +0000 Subject: [PATCH 02/46] hermes: add a fail-closed full-handoff acceptance harness Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report. --- docs/hermes_full_handoff_acceptance.md | 323 ++++++++++++++ scripts/ops/hermes_handoff_acceptance.py | 236 +++++++++++ scripts/ops/hermes_handoff_catalog.py | 189 +++++++++ scripts/ops/hermes_handoff_checks_access.py | 234 +++++++++++ scripts/ops/hermes_handoff_checks_delivery.py | 373 +++++++++++++++++ scripts/ops/hermes_handoff_checks_platform.py | 335 +++++++++++++++ scripts/ops/hermes_handoff_checks_workers.py | 388 +++++++++++++++++ scripts/ops/hermes_handoff_ephemeral.py | 313 ++++++++++++++ scripts/ops/hermes_handoff_evaluators.py | 266 ++++++++++++ scripts/ops/hermes_handoff_exec.py | 252 +++++++++++ scripts/ops/hermes_handoff_json_rules.py | 335 +++++++++++++++ scripts/ops/hermes_handoff_model.py | 270 ++++++++++++ scripts/ops/hermes_handoff_policy.py | 377 +++++++++++++++++ scripts/ops/hermes_handoff_redaction.py | 223 ++++++++++ scripts/ops/hermes_handoff_rules.py | 159 +++++++ scripts/ops/hermes_handoff_run.py | 334 +++++++++++++++ .../tests/test_hermes_handoff_acceptance.py | 395 ++++++++++++++++++ testing/tests/test_hermes_handoff_catalog.py | 194 +++++++++ .../tests/test_hermes_handoff_ephemeral.py | 271 ++++++++++++ .../tests/test_hermes_handoff_evaluators.py | 201 +++++++++ testing/tests/test_hermes_handoff_exec.py | 173 ++++++++ .../tests/test_hermes_handoff_json_rules.py | 279 +++++++++++++ testing/tests/test_hermes_handoff_model.py | 180 ++++++++ testing/tests/test_hermes_handoff_policy.py | 159 +++++++ .../tests/test_hermes_handoff_redaction.py | 162 +++++++ testing/tests/test_hermes_handoff_rules.py | 135 ++++++ testing/tests/test_hermes_handoff_run.py | 320 ++++++++++++++ testing/tests/test_hermes_handoff_support.py | 110 +++++ 28 files changed, 7186 insertions(+) create mode 100644 docs/hermes_full_handoff_acceptance.md create mode 100755 scripts/ops/hermes_handoff_acceptance.py create mode 100644 scripts/ops/hermes_handoff_catalog.py create mode 100644 scripts/ops/hermes_handoff_checks_access.py create mode 100644 scripts/ops/hermes_handoff_checks_delivery.py create mode 100644 scripts/ops/hermes_handoff_checks_platform.py create mode 100644 scripts/ops/hermes_handoff_checks_workers.py create mode 100644 scripts/ops/hermes_handoff_ephemeral.py create mode 100644 scripts/ops/hermes_handoff_evaluators.py create mode 100644 scripts/ops/hermes_handoff_exec.py create mode 100644 scripts/ops/hermes_handoff_json_rules.py create mode 100644 scripts/ops/hermes_handoff_model.py create mode 100644 scripts/ops/hermes_handoff_policy.py create mode 100644 scripts/ops/hermes_handoff_redaction.py create mode 100644 scripts/ops/hermes_handoff_rules.py create mode 100644 scripts/ops/hermes_handoff_run.py create mode 100644 testing/tests/test_hermes_handoff_acceptance.py create mode 100644 testing/tests/test_hermes_handoff_catalog.py create mode 100644 testing/tests/test_hermes_handoff_ephemeral.py create mode 100644 testing/tests/test_hermes_handoff_evaluators.py create mode 100644 testing/tests/test_hermes_handoff_exec.py create mode 100644 testing/tests/test_hermes_handoff_json_rules.py create mode 100644 testing/tests/test_hermes_handoff_model.py create mode 100644 testing/tests/test_hermes_handoff_policy.py create mode 100644 testing/tests/test_hermes_handoff_redaction.py create mode 100644 testing/tests/test_hermes_handoff_rules.py create mode 100644 testing/tests/test_hermes_handoff_run.py create mode 100644 testing/tests/test_hermes_handoff_support.py diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md new file mode 100644 index 00000000..8b9103c2 --- /dev/null +++ b/docs/hermes_full_handoff_acceptance.md @@ -0,0 +1,323 @@ +# Hermes full-handoff acceptance harness and release runbook + +`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform +handoff is fit to release. It is read-only by default, it classifies every check +`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or +`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up +to evidence of absence: a probe that could not run, a catalog entry whose +evidence no longer exists, an expired deadline, or a harness bug all land in +`NOT_RUN` and stop the release. + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --node-count 21 \ + --expected-suspension flux-system/bstein-dev-home-migrations \ + --output build/hermes-handoff-acceptance.json +``` + +Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to +`--output` (or stdout); a short human summary goes to stderr unless +`--json-only` is passed. + +## Why two vantages + +Every authority claim is checked from two places that cannot cover for each +other: + +| Vantage | What it is | What it proves | +| --- | --- | --- | +| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | +| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | +| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | + +Two rules keep this honest: + +* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and + `--as-uid` are rejected by policy for any operator-side command. They are + reachable only as the *inner* command of a `kubectl exec`, where the identity + is testing its own refusal to impersonate rather than borrowing authority. + `kubectl auth can-i --as` is therefore not available as a substitute for a + real self-probe. +* **The two vantages must be different principals.** Before any check runs, the + harness asks each vantage `kubectl auth whoami` and compares. If the operator + and the in-pod probe authenticate as the same principal, the dual-vantage + evidence is a restatement rather than a corroboration, and the run is `NO_GO` + with `the operator and in-pod vantages authenticate as the same principal`. + Run the harness from a genuinely external kubeconfig — not from inside the + Hermes owner pod. + +Deny checks additionally require a **real refused request**, not only an +authorization review. `kubectl auth can-i` reports what RBAC says; only an +attempt proves the API server enforces it. Mutating attempts go through +`--dry-run=server`: the API server authorises the request and then discards it, +so a refusal is genuine and nothing is written either way. + +## Safety properties + +These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is +a policy error before a run rather than a mutation or a leak during one. + +* **No mutation in the default mode.** Only read subcommands are reachable; + `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run + flag. `git push` and every non-`GET` forge method are refused outside the + armed mode. There is deliberately no live TokenRequest probe — a successful + one would mint a real credential. +* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, + `curl`, and `wget` are not on the binary allowlist. Secrets may only be read + with `-o name`. Environment probes list variable *names* and never values. + Provider health files are read through a frozen template that projects an + allow-listed set of non-secret status fields. The node account's lock state is + reported without reading its password hash; its authorized keys are counted, + never printed. +* **Shell is frozen.** `sh -c` is reachable only through the reviewed, + parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain + shell metacharacters. +* **Output is bounded and screened.** Each capture is truncated to a byte budget + *before* it is screened, so a large dump cannot outrun the screen. Bulk + evidence (the routing-log tail, the Kanban listing) is parsed but replaced with + `[output not recorded]` in the artifact. The screen preserves JSON quoting, so + screened structured output is still parsable. +* **The report is re-checked before it is published.** `unscreened_fields` walks + the rendered payload; if anything still looks credential-shaped the report is + replaced with a `NO_GO` stub naming the offending pointers, and the file that + would have leaked is never written. + +## What the catalog asserts + +The catalog is declarative (`hermes_handoff_catalog.py` plus the four +`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not +how it is plumbed. 71 checks in 14 groups: + +| Group | Assertions | +| --- | --- | +| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | +| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | +| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | +| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | +| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | +| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | +| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | +| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | +| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | +| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | +| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | +| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | +| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | +| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | + +### Site configuration + +A few checks are emitted only when the operator declares what to expect, +because a `names_present` assertion over an empty expectation would pass on any +workload at all — which reads as coverage without being any: + +| Flag | Enables | +| --- | --- | +| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | +| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | +| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | + +`--node-count` must be the real node count; the default of `1` makes the +fleet-coverage check vacuous. + +### Catalog drift is a NO_GO, not a pass + +If a workload, field, or manifest is renamed, the affected checks report +`NOT_RUN` because the evidence they name is no longer there. That is the +intended behaviour: the release stops until either the platform or the catalog +is corrected. Do not "fix" a `NOT_RUN` by deleting the check. + +## Ephemeral mutation mode + +Proving that a worker can still open a draft pull request means actually opening +one. That is the only part of the harness that writes anything, and it is a +separate mode: + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --arm-ephemeral-push \ + --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ + --ephemeral-token acceptance-20260817a \ + --output build/hermes-handoff-acceptance-armed.json +``` + +* The confirmation phrase must match exactly. +* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be + unique per run; the ref is always + `ephemeral/hermes-handoff-acceptance/`. +* **Preflight runs before any network call**: wrong phrase, wrong repository, + malformed token, or a protected push target all abort with `arming refused` + and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, + `production`, `stable`, `trunk`, `prod`, and `default` are refused by name + (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral + acceptance ref is refused by pattern. +* The guard is **exercised on every armed run** by + `ephemeral.protected-branch-refusal`, rather than trusting that it was checked + once. +* The push target is guarded; the pull-request *base* is not. Opening a draft + pull request against `main` writes nothing to `main`. The draft title always + starts with `WIP: `. +* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to + confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed + branch fails the run rather than passing it. Cleanup runs even when an earlier + step failed. +* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` + are refused even in armed mode. + +A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so +a read-only sweep can still reach `GO`. + +## Release order + +### Baseline, already merged and deployed + +PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as +`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is +present; `baseline.origin-main-descends-merged-work` asserts it. Override with +`--baseline-commit` only when a later merge legitimately replaces it. + +### Pending work, observed 2026-08-17T10:07Z + +These heads move. The harness re-fetches them at run time via +`baseline.pending-pull-requests-recorded`; the table is the snapshot this +document was written against, not an input. + +| PR | Branch | Head at observation | State | Delivers | +| --- | --- | --- | --- | --- | +| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | +| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | +| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | +| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | +| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | + +Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. + +### Merge order and why + +1. **#14 first.** It is the only change that *reduces* authority: it deletes + `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces + it with the namespaced/cluster observer roles, and it hardens the node + account. Merging it last would mean every earlier acceptance run measured a + cluster-admin agent. Every `access-denied` and `nodes` check depends on it. +2. **#16 second.** The image lane must exist before anything that needs a new + agent image is rolled out. It also touches `testing/quality_contract.json`, + so merging it early keeps later rebases small. `build.*` depends on it. +3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 + also touches — take #16's shape and rebase #15 onto it. `reliability.*` + depends on it. +4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` + and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 + also touch. Merging it after them avoids resolving the same conflict twice. +5. **#18 last.** The pool depends on the bounded forge client from #14, the + execution-safety patch from #15, and the agent image from #16. `pool.*` + depends on it. + +Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` +(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), +`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), +`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), +`dockerfiles/Dockerfile.hermes-agent` (#15, #16), +`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), +`testing/tests/test_hermes_runtime_access.py` (#16, #18). + +PR #12 (AI usage exporter) is independent and can merge at any point; it touches +`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. + +### Image build and Flux rollout + +1. Merge in the order above; observe the default-branch Jenkins build to a + terminal result after each merge + (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). +2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an + immutable `git--build-` tag and emits + `build/hermes-agent-release/hermes-image-update.patch`. +3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, + and merge. The digest is what pins the deployment; + `build.deployed-image-is-digest-pinned` asserts it. +4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` + and `hermes-chat` are `Ready`, then wait for the rollout to settle. +5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an + in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled + deployment will legitimately fail the `identity`, `routing`, and `surfaces` + groups. + +### Rollback points + +| After | Roll back by | Restores | +| --- | --- | --- | +| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | +| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | +| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | +| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | +| #17 merge | Revert the merge | Previous lane concurrency and node placement | +| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | + +Every rollback is a Git revert reconciled by Flux. Do not roll back by direct +`kubectl` edit; the next reconcile would undo it. + +## Go / no-go checklist + +Run in order. Stop at the first item that does not hold. + +1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. +2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each + with a green default-branch build. +3. The agent image has been built by the #16 lane, is pinned by digest in + `services/hermes/kustomization.yaml`, and Flux has finished reconciling. +4. The harness is run from an **external operator kubeconfig**, not from inside + the Hermes owner pod, with `--node-count` set to the real node count and every + deliberate Flux suspension declared via `--expected-suspension`. +5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory + `NOT_RUN`, no `harness_errors`. +6. The armed run reports `PASS` for all four `ephemeral.*` checks, including + `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are + confirmed gone. +7. The JSON report is archived with the exact `origin/main` commit and the + recorded pull-request heads from `baseline.pending-pull-requests-recorded`. + +Only then is the handoff releasable. + +## Known limits + +Stated so nobody mistakes a proxy for the thing itself. + +* **Merge, approve, and close authority is asserted, not exercised.** The + harness never attempts a merge. It proves the forge identity holds + `permissions.admin: false` and is live-refused on an administrative route; + the route-level allowlist is covered by PR #14's own regression suite. +* **Attach and port-forward have no side-effect-free live attempt.** Their + authorization reviews ride alongside a real, refused control-plane write in the + same check, and pod exec — the one subresource with a harmless real attempt — + has its own check. +* **TokenRequest is asserted by review.** A successful live attempt would mint a + real credential, so the check pairs the `serviceaccounts/token` review with a + refused `serviceaccount` create in the same RBAC family. +* **Branch protection is not read.** The harness identity is not a repository + administrator, so `GET /branch_protections` is refused — which is itself + recorded as evidence. Protected-push refusal is guaranteed by the harness's own + preflight, exercised on every armed run, not by reading the forge's config. +* **Stale/duplicate assignment safety is asserted as deployed configuration.** + The behaviour itself is covered by the pool's regression suite; the harness + checks that the workers carry the configuration and that the coordinator keeps + sole ownership of its claims. +* **Telegram session continuity is per tenant** and depends on there having been + Telegram traffic, so it is off by default. The always-on check is that topic + and tenant state live on a persistent claim rather than in the pod. +* **The routing window is a tail**, 600 records by default. A property that did + not occur inside that window is reported as missing rather than absent; widen + the window or wait for traffic rather than removing the assertion. + +## Tests + +``` +python3 -m pytest testing/tests/ -k handoff +python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ + -m pytest testing/tests/ -k handoff && python3 -m coverage report +``` + +The suite covers parsing, redaction, timeouts and deadlines, partial failure, +zero state, dual-vantage mismatch, mandatory skips, catalog structure, and +ephemeral arming and cleanup — including the cases where cleanup cannot be +verified. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py new file mode 100755 index 00000000..7be6de29 --- /dev/null +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Fail-closed acceptance harness for the Hermes full handoff. + +Read-only by default. The harness collects evidence from two genuinely separate +places — an external operator vantage and the Hermes agent probing itself from +inside its own pod — classifies every check PASS / FAIL / NOT_RUN / +NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary. + +Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an +unreachable vantage, a catalog whose evidence no longer exists, an expired +deadline, or two vantages that turn out to be the same principal. Nothing here +rounds an absence of evidence up to a pass. + + scripts/ops/hermes_handoff_acceptance.py --context atlas-operator + scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + +Mutation lives behind a separate arming flag, needs an exact confirmation +phrase and a unique ref, refuses protected push targets before any network +call, and verifies its own cleanup. A default run reports those checks NOT_RUN. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from hermes_handoff_catalog import Targets # noqa: E402 +from hermes_handoff_ephemeral import ( # noqa: E402 + CONFIRMATION, + ArmingError, + ArmRequest, + preflight, + run_armed, +) +from hermes_handoff_exec import ( # noqa: E402 + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_RUN_DEADLINE, + SELF, + Runner, +) +from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 +from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 +from hermes_handoff_run import ( # noqa: E402 + build_catalog, + build_report, + resolve_vantages, + unavailable_report, +) + +EXIT_GO = 0 +EXIT_NO_GO = 1 +EXIT_USAGE = 2 + + +def build_parser() -> argparse.ArgumentParser: + """Return the command-line interface for the harness.""" + parser = argparse.ArgumentParser( + prog="hermes_handoff_acceptance", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") + parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") + parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") + parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") + parser.add_argument( + "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + ) + parser.add_argument( + "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + ) + parser.add_argument( + "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + ) + parser.add_argument( + "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + ) + parser.add_argument( + "--dependency-pr", + type=int, + action="append", + default=None, + help="Pull request that must be merged before this release is certifiable (repeatable).", + ) + parser.add_argument( + "--expected-suspension", + action="append", + default=None, + metavar="NAMESPACE/NAME", + help="Flux object allowed to be suspended (repeatable).", + ) + parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") + parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") + parser.add_argument( + "--chat-config-revision", default="", help="Expected chat pod-template config revision." + ) + parser.add_argument( + "--chat-ordinal", + type=int, + default=0, + help="Chat tenant ordinal that carries the Telegram identity.", + ) + parser.add_argument( + "--expect-telegram-sessions", + action="store_true", + help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", + ) + parser.add_argument( + "--pool-worker-env", + action="append", + default=None, + help="Environment variable the pool workers must declare (repeatable).", + ) + parser.add_argument( + "--max-evidence-age", + type=int, + default=86_400, + help="Seconds before provider and routing evidence is treated as stale.", + ) + armed = parser.add_argument_group("ephemeral mutation mode") + armed.add_argument( + "--arm-ephemeral-push", + action="store_true", + help="Permit one ephemeral branch push and draft pull request, then remove both.", + ) + armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") + armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") + armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") + armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") + armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + return parser + + +def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets: + """Return the catalog targets a parsed command line describes.""" + defaults = Targets() + return Targets( + namespace=arguments.namespace, + baseline_commit=arguments.baseline_commit, + dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + expected_suspensions=tuple(arguments.expected_suspension or ()), + node_count=arguments.node_count, + pool_replicas=arguments.pool_replicas, + chat_config_revision=arguments.chat_config_revision, + chat_ordinal=arguments.chat_ordinal, + expect_telegram_sessions=arguments.expect_telegram_sessions, + pool_worker_env=tuple(arguments.pool_worker_env or ()), + max_evidence_age_seconds=arguments.max_evidence_age, + repo=arguments.repo, + remote=arguments.remote, + now=now, + ) + + +def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]: + """Write the report, refusing to publish one that still looks credential-shaped.""" + payload = report.as_dict() + offenders = unscreened_fields(payload) + if offenders: + payload = { + "harness": payload["harness"], + "decision": "NO_GO", + "harness_errors": [ + "the rendered report failed its own credential screening at " + + ", ".join(offenders[:8]) + ], + } + rendered = json.dumps(payload, indent=2, sort_keys=True, default=str) + if arguments.output: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered, file=stream or sys.stdout) + if not arguments.json_only: + print(report.render_summary(), file=sys.stderr) + return offenders + + +def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: + request = ArmRequest( + repo=arguments.repo, + remote=arguments.remote, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=arguments.ephemeral_base, + ) + preflight(request, arguments.repo) + vantage = vantages.get(SELF) or vantages["operator"] + return run_armed(runner, vantage, request) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the acceptance sweep and return the process exit status.""" + arguments = build_parser().parse_args(argv) + now = dt.datetime.now(dt.timezone.utc) + started_at = utc_now(lambda: now) + targets = targets_from_args(arguments, now) + mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY + + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + ) + vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) + + armed_results: list = [] + if arguments.arm_ephemeral_push: + try: + armed_results = _armed_results(arguments, runner, vantages) + except ArmingError as exc: + report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) + emit(report, arguments) + return EXIT_NO_GO + + specs = build_catalog(targets) + report = build_report( + runner, targets, specs, vantages, records, mode, started_at, armed_results + ) + offenders = emit(report, arguments) + if offenders: + return EXIT_NO_GO + return EXIT_GO if report.decision == GO else EXIT_NO_GO + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py new file mode 100644 index 00000000..b59ff2b9 --- /dev/null +++ b/scripts/ops/hermes_handoff_catalog.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Declarative acceptance catalog: baseline, identity, routing, scopes, access. + +The catalog is data on purpose. A reviewer should be able to read what the +release is asserted to satisfy without reading any plumbing, and the harness +should be able to prove structural properties over every entry — both vantages +represented, no impersonation from outside a pod, no mutation outside a server +dry run — before it runs a single command. + +Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload +is renamed, the affected checks report ``NOT_RUN`` because the evidence they +name is not there, and the release stops until either the platform or the +catalog is corrected. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step + +OPERATOR = "operator" +SELF = "self" +SWITCHYARD = "switchyard" +NODE = "node" +CHAT = "chat" +VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) + +# `namespace/namesuspendType=Status,...` — one compact row per object. +FLUX_PROJECTION = ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" +).replace("'", '"') + +ENV_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" +).replace("'", '"') + +INIT_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" +).replace("'", '"') + +NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') + +PROVIDER_API_KEY_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AZURE_OPENAI_API_KEY", + "CLAUDE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) +FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") + +CODEX_HEALTH = "/opt/data/provider-health/codex.json" +CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" +CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at" +CLAUDE_HEALTH_FIELDS = ( + "authenticated,api_provider,auth_method,transport,state,subscription_type," + "model,effort,latency_ms,checked_at" +) +ROUTING_LOG = "/var/lib/switchyard/routing.jsonl" + + +@dataclass(frozen=True) +class Targets: + """Runtime names and expected values the catalog is written against. + + Everything a site can legitimately rename lives here so the assertions stay + readable and a rename is a configuration change rather than a code change. + """ + + namespace: str = "hermes" + agent_deployment: str = "hermes-agent" + agent_container: str = "hermes" + chat_statefulset: str = "hermes-chat-tenant" + chat_container: str = "hermes" + chat_ordinal: int = 0 + chat_router_deployment: str = "hermes-chat-router" + triage_deployment: str = "hermes" + triage_container: str = "hermes" + switchyard_deployment: str = "hermes-switchyard" + switchyard_container: str = "switchyard" + node_daemonset: str = "hermes-node-ssh-access" + node_container: str = "key-reconciler" + pool_statefulset: str = "hermes-execution-worker" + pool_replicas: int = 3 + pool_selector: str = "app=hermes-execution-worker" + coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + node_account: str = "hermes-agent" + node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" + node_count: int = 1 + repo: str = "atlas/titan-iac" + remote: str = "origin" + baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" + dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + expected_suspensions: tuple[str, ...] = () + harbor_namespace: str = "harbor" + harbor_immutability_job: str = "hermes-agent-immutability" + builder_namespace: str = "jenkins" + builder_serviceaccount: str = "hermes-image-builder" + builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" + builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) + chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) + chat_config_revision: str = "" + pool_worker_env: tuple[str, ...] = () + max_evidence_age_seconds: int = 86_400 + routing_tail_lines: int = 600 + routing_tail_bytes: int = 512 * 1024 + listing_bytes: int = 512 * 1024 + git_askpass: str = "/opt/coordinator/gitea_askpass.sh" + expect_telegram_sessions: bool = False + now: Any = None + extra: dict[str, Any] = field(default_factory=dict) + + +def step( + key: str, + vantage: str, + *argv: str, + kind: str = READ, + optional: bool = False, + record: bool = True, + max_bytes: int | None = None, +) -> Step: + """Return a probe step addressed to one vantage.""" + return Step( + key=key, + vantage=vantage, + argv=tuple(argv), + kind=kind, + optional=optional, + record=record, + max_bytes=max_bytes, + ) + + +def check( + identifier: str, + title: str, + group: str, + rule: str, + steps: Sequence[Step], + expect: dict[str, Any] | None = None, + mandatory: bool = True, + scope: str = ALWAYS, + rationale: str = "", +) -> CheckSpec: + """Return one catalog entry.""" + return CheckSpec( + id=identifier, + title=title, + group=group, + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + scope=scope, + rationale=rationale, + ) + + +def deny( + identifier: str, + title: str, + review: Sequence[str], + attempt: Sequence[str], + rationale: str, + group: str = "access-denied", +) -> CheckSpec: + """Return a deny check that pairs a self review with a real refused request.""" + return check( + identifier, + title, + group, + "denied", + [ + step("review", SELF, *review, kind=REVIEW), + step("attempt", SELF, *attempt, kind=ATTEMPT), + ], + rationale=rationale, + ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py new file mode 100644 index 00000000..a863e0f3 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Acceptance checks for what Hermes may and may not do to the cluster. + +Every deny check pairs an in-pod authorization review with a real request that +must be refused, because a review reports what RBAC says and only an attempt +proves the API server enforces it. Mutating attempts go through a server dry +run: the API server authorises the request and discards it, so the refusal is +genuine and nothing is written either way. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FLUX_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + deny, + step, +) +from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec +from hermes_handoff_policy import shell + + +def _access(targets: Targets) -> list[CheckSpec]: + namespace = targets.namespace + return [ + check( + "access.no-cluster-admin-binding-for-agent", + "No ClusterRoleBinding grants the agent service account cluster-admin", + "access-denied", + "names_absent", + [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, + rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + ), + deny( + "access.secrets-are-denied", + "Hermes cannot read Secrets cluster-wide", + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + ), + check( + "access.service-account-tokens-are-denied", + "Hermes cannot mint a service-account token", + "access-denied", + "denied", + [ + step( + "review-token", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts/token", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "review-serviceaccount", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + namespace, + "create", + "serviceaccount", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + ), + deny( + "access.impersonation-is-denied", + "Hermes cannot impersonate another identity", + ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), + ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), + "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + ), + deny( + "access.workload-mutation-is-denied", + "Hermes cannot mutate its own workload", + ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), + ( + "kubectl", + "--namespace", + namespace, + "patch", + f"deploy/{targets.agent_deployment}", + "--type=merge", + "--patch", + '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', + "--dry-run=server", + ), + "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + ), + deny( + "access.pod-exec-is-denied", + "Hermes cannot exec into an arbitrary pod", + ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), + ( + "kubectl", + "--namespace", + namespace, + "exec", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--", + *shell("path_readable", path="/tmp"), + ), + "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + ), + check( + "access.attach-portforward-and-kube-system-writes-are-denied", + "Hermes holds no attach, port-forward, or control-plane write authority", + "access-denied", + "denied", + [ + step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), + step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), + step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + "kube-system", + "create", + "configmap", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + ), + check( + "access.required-reads-are-allowed", + "Hermes retains the reads its operator role depends on", + "access-allowed", + "allowed", + [ + step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), + step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), + ], + ), + check( + "access.pod-logs-are-allowed", + "Hermes can still read pod logs", + "access-allowed", + "allowed", + [ + step( + "logs", + SELF, + "kubectl", + "--namespace", + targets.namespace, + "logs", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--tail", + "1", + record=False, + ) + ], + ), + check( + "access.flux-and-helm-status-are-allowed", + "Hermes can read Flux and Helm reconciliation status", + "access-allowed", + "allowed", + [ + step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + ], + ), + check( + "access.namespace-view-agrees-across-vantages", + "The operator and in-pod vantages see the same namespace inventory", + "access-allowed", + "vantages_agree", + [ + step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), + step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + ], + {"steps": ("operator", "self")}, + rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", + ), + ] + + +def _gitops(targets: Targets) -> list[CheckSpec]: + return [ + check( + f"gitops.{kind.split('.')[0]}-reconcile-cleanly", + f"No unexpected {label} suspension or unhealthy reconciliation", + "gitops", + "flux_health", + [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + {"step": "objects", "expected_suspensions": targets.expected_suspensions}, + ) + for kind, label in ( + ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"), + ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"), + ) + ] + + +def access_checks(targets: Targets) -> list[CheckSpec]: + """Return the access-denied, access-allowed, and GitOps health checks.""" + return [*_access(targets), *_gitops(targets)] diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py new file mode 100644 index 00000000..a3dbaad7 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Acceptance checks for forge authority, node accounts, images, and regressions. + +What the platform is allowed to do outside its own namespace: which forge +authority a worker holds, what a node account is and is not, how an image is +built and pinned, and whether the fixes the release depends on are actually +running rather than merely merged. + +Where a property has no side-effect-free live probe — a merge that must never be +attempted, a branch protection the harness identity cannot read — the entry says +so in its rationale and asserts the nearest fact that *is* observable, rather +than quietly asserting nothing. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FORGE_CREDENTIAL_NAMES, + INIT_NAME_PROJECTION, + NODE, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import ATTEMPT, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + +HOST_ETC = "/host-etc" +HOST_HOME = "/host-home" + + +def _forge(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + return [ + check( + "forge.worker-holds-no-git-credential-variable", + "No forge credential is exported into the worker process", + "forge", + "names_absent", + [step("env", SELF, *shell("env_names"))], + {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, + rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", + ), + check( + "forge.worker-holds-no-credential-file", + "The worker home has no stored Git credential file", + "forge", + "stdout_matches", + [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], + {"step": "credentials", "equals": "unreadable"}, + ), + check( + "forge.identity-is-not-an-administrator", + "The forge identity the worker uses holds no administrative rights", + "forge", + "json_field", + [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")], + {"step": "user", "fields": {"is_admin": False}}, + ), + check( + "forge.repository-authority-is-push-only", + "The worker can push a branch but cannot administer the repository", + "forge", + "json_field", + [step("repo", SELF, GITEA_CLIENT, "GET", api)], + { + "step": "repo", + "fields": { + "permissions.admin": False, + "permissions.push": True, + "permissions.pull": True, + }, + }, + rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.", + ), + check( + "forge.administrative-route-is-refused", + "An administrative forge route is refused for this identity", + "forge", + "denied", + [ + step( + "attempt", + SELF, + GITEA_CLIENT, + "GET", + f"{api}/branch_protections", + kind=ATTEMPT, + ) + ], + rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", + ), + check( + "forge.clone-and-fetch-still-work", + "The worker can still read the repository it is expected to work in", + "forge", + "allowed", + [ + step( + "ls-remote", + SELF, + *shell( + "git_ls_remote", + askpass=targets.git_askpass, + url=f"https://scm.bstein.dev/{targets.repo}.git", + ), + ) + ], + ), + ] + + +def _nodes(targets: Targets) -> list[CheckSpec]: + account = targets.node_account + return [ + check( + "nodes.account-password-is-locked", + "The dedicated node account has no usable password", + "nodes", + "stdout_matches", + [ + step( + "lock", + NODE, + *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), + ) + ], + {"step": "lock", "equals": "locked"}, + rationale="The probe reports the lock marker only; the password hash is never read.", + ), + check( + "nodes.account-identity-is-dedicated", + "The node account is the dedicated Hermes identity, not a shared one", + "nodes", + "stdout_matches", + [ + step( + "identity", + NODE, + *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), + ) + ], + {"step": "identity", "equals": targets.node_account_identity}, + ), + check( + "nodes.account-holds-no-supplementary-groups", + "The node account joins no supplementary or administrative group", + "nodes", + "stdout_matches", + [ + step( + "groups", + NODE, + *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), + ) + ], + {"step": "groups", "equals": ""}, + rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", + ), + check( + "nodes.account-has-no-sudo-policy", + "No sudoers fragment names the node account", + "nodes", + "stdout_matches", + [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], + {"step": "sudoers", "equals": "absent"}, + ), + check( + "nodes.account-holds-a-single-authorized-key", + "Exactly one authorized key is installed for the node account", + "nodes", + "stdout_matches", + [ + step( + "keys", + NODE, + *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), + ) + ], + {"step": "keys", "equals": "1"}, + rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", + ), + *[ + check( + f"nodes.{label}-is-root-only", + f"{title} remains unreadable to the node account", + "nodes", + "stdout_matches", + [step("mode", NODE, *shell("path_mode", path=path))], + {"step": "mode", "equals": expected}, + ) + for label, title, path, expected in ( + ( + "k3s-credentials", + "The K3s admin kubeconfig", + f"{HOST_ETC}/rancher/k3s/k3s.yaml", + ["600 root:root", "absent"], + ), + ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), + ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), + ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), + ) + ], + check( + "nodes.hardening-covers-every-node", + "The node account audit runs on every node in the cluster", + "nodes", + "distinct_count", + [ + step( + "coverage", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + f"app={targets.node_daemonset}", + "-o", + NODE_NAME_PROJECTION, + ) + ], + {"step": "coverage", "minimum": targets.node_count}, + rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", + ), + ] + + +def _build(targets: Targets) -> list[CheckSpec]: + pipeline = f"{targets.remote}/main:{targets.builder_pipeline}" + return [ + check( + "build.builder-service-account-is-tokenless", + "The image builder service account mounts no API token", + "build", + "json_field", + [ + step( + "serviceaccount", + OPERATOR, + "kubectl", + "--namespace", + targets.builder_namespace, + "get", + "serviceaccount", + targets.builder_serviceaccount, + "-o", + "json", + ) + ], + {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + ), + check( + "build.builder-runs-unprivileged-with-exact-capabilities", + "The build pod is unprivileged and adds only the expected capabilities", + "build", + "names_present", + [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)], + { + "step": "pipeline", + "names": ( + targets.builder_capabilities, + "privileged: false", + "allowPrivilegeEscalation: false", + "automountServiceAccountToken: false", + 'drop: ["ALL"]', + ), + }, + rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.", + ), + check( + "build.harbor-immutability-rule-is-applied", + "The Harbor immutable-tag rule for the agent image has been applied", + "build", + "json_numeric", + [ + step( + "job", + OPERATOR, + "kubectl", + "--namespace", + targets.harbor_namespace, + "get", + "job", + targets.harbor_immutability_job, + "-o", + "json", + ) + ], + {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", + ), + check( + "build.deployed-image-is-digest-pinned", + "The running agent image is pinned by digest, not by a mutable tag", + "build", + "names_present", + [ + step( + "image", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', + ) + ], + {"step": "image", "contains": ("@sha256:",)}, + ), + ] + + +def _reliability(targets: Targets) -> list[CheckSpec]: + return [ + check( + "reliability.finalization-and-replay-patches-are-deployed", + "The agent workload runs the finalization, replay, and session patches", + "reliability", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.agent_init_containers}, + rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.", + ), + check( + "reliability.regression-suites-are-present-on-main", + "The decomposition and worker-recovery regression suites are on main", + "reliability", + "names_present", + [ + step( + "suites", + OPERATOR, + "git", + "ls-tree", + "--name-only", + f"{targets.remote}/main", + "testing/tests/", + ) + ], + { + "step": "suites", + "names": ( + "testing/tests/test_hermes_cli_lanes.py", + "testing/tests/test_hermes_coordinator.py", + "testing/tests/test_hermes_worker_recovery.py", + ), + }, + ), + ] + + +def delivery_checks(targets: Targets) -> list[CheckSpec]: + """Return the forge, node, build, and reliability checks.""" + return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py new file mode 100644 index 00000000..d1986976 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the release baseline, provider identity, and routing. + +These are the facts a handoff rests on before any authority question is asked: +that the tree under test is the merged one, that both providers authenticate +through the subscriptions they are supposed to, that Switchyard actually decided +the routes it claims to, and that chat, agent, and triage are still three +distinct scopes rather than one surface wearing three hostnames. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CLAUDE_HEALTH, + CLAUDE_HEALTH_FIELDS, + CODEX_HEALTH, + CODEX_HEALTH_FIELDS, + ENV_NAME_PROJECTION, + OPERATOR, + PROVIDER_API_KEY_NAMES, + ROUTING_LOG, + SELF, + SWITCHYARD, + Targets, + check, + step, +) +from hermes_handoff_model import CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + + +def _baseline(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + checks = [ + check( + "baseline.origin-main-descends-merged-work", + "origin/main descends from the merged Hermes isolation baseline", + "baseline", + "allowed", + [ + step( + "ancestor", + OPERATOR, + "git", + "merge-base", + "--is-ancestor", + targets.baseline_commit, + f"{targets.remote}/main", + ) + ], + rationale="Everything downstream assumes the merged worker-isolation baseline is present.", + ), + check( + "baseline.pending-pull-requests-recorded", + "Exact open pull-request heads are recorded, not assumed", + "baseline", + "json_record", + [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + { + "step": "open", + "record": { + "numbers": "[].number", + "head_refs": "[].head.ref", + "head_shas": "[].head.sha", + "base_shas": "[].base.sha", + "drafts": "[].draft", + "mergeable": "[].mergeable", + }, + }, + rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.", + ), + check( + "baseline.operator-checkout-clean", + "The operator checkout has no uncommitted drift", + "baseline", + "stdout_matches", + [step("status", OPERATOR, "git", "status", "--porcelain")], + {"step": "status", "equals": ""}, + mandatory=False, + rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.", + ), + ] + checks += [ + check( + f"baseline.dependency-pr-{number}-merged", + f"Pull request #{number} is merged into main", + "baseline", + "json_field", + [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], + {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", + ) + for number in targets.dependency_pull_requests + ] + return checks + + +def _identity(targets: Targets) -> list[CheckSpec]: + codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS) + claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS) + agent = f"deploy/{targets.agent_deployment}" + return [ + check( + "identity.codex-is-chatgpt-subscription", + "Codex authenticates through the ChatGPT subscription, not an API key", + "identity", + "json_field", + [step("health", SELF, *codex)], + { + "step": "health", + "fields": { + "authenticated": True, + "transport": "codex-chatgpt-subscription", + "state": "available", + }, + }, + rationale="Only allow-listed status fields are read; the credential itself is never opened.", + ), + check( + "identity.codex-evidence-is-fresh", + "Codex authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *codex)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + rationale="Stale provider evidence describes a state the release no longer has.", + ), + check( + "identity.claude-is-firstparty-subscription", + "Claude authenticates as a claude.ai first-party subscription", + "identity", + "json_field", + [step("health", SELF, *claude)], + { + "step": "health", + "fields": { + "authenticated": True, + "api_provider": "firstParty", + "auth_method": "claude.ai", + "transport": "claude-code-cli-subscription", + "state": "available", + }, + }, + ), + check( + "identity.claude-evidence-is-fresh", + "Claude authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *claude)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + ), + check( + "identity.no-provider-api-key-in-pod", + "No provider API-key variable is set in the running agent process", + "identity", + "names_absent", + [step("env", SELF, *shell("env_names"))], + { + "step": "env", + "names": PROVIDER_API_KEY_NAMES, + "contains": ("API_KEY",), + }, + rationale="Names only. The probe lists variable names and never their values.", + ), + check( + "identity.no-provider-api-key-in-manifest", + "No provider API-key variable is declared on the agent workload", + "identity", + "names_absent", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + agent, + "-o", + ENV_NAME_PROJECTION, + ) + ], + {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)}, + rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.", + ), + ] + + +def _routing(targets: Targets) -> list[CheckSpec]: + switchyard = f"deploy/{targets.switchyard_deployment}" + tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines)) + routing_step = step( + "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes + ) + return [ + check( + "routing.switchyard-is-available", + "Switchyard has a ready replica serving every model-call boundary", + "routing", + "json_numeric", + [ + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + switchyard, + "-o", + "json", + ) + ], + {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + ), + check( + "routing.provider-effort-and-fallback-evidence", + "Routing evidence covers both providers, all graded efforts, and a real fallback", + "routing", + "routing_evidence", + [routing_step], + { + "step": "routing", + "providers": ("codex", "claude"), + "efforts": ("medium", "high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + "max_age_seconds": targets.max_evidence_age_seconds, + "now": targets.now, + }, + rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.", + ), + check( + "routing.codex-latency-recorded", + "Codex boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + check( + "routing.claude-latency-recorded", + "Claude boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + ] + + +def _scope_profile( + targets: Targets, name: str, workload: str, container: str, expected: str +) -> CheckSpec: + projection = ( + f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' + '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' + ) + return check( + f"scopes.{name}-profile-is-distinct", + f"The {name} surface runs under its own routing profile", + "scopes", + "stdout_matches", + [ + step( + "profile", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + workload, + "-o", + projection, + ) + ], + {"step": "profile", "equals": expected}, + rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.", + ) + + +def _scopes(targets: Targets) -> list[CheckSpec]: + return [ + _scope_profile( + targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + ), + _scope_profile( + targets, + "chat", + f"statefulset/{targets.chat_statefulset}", + targets.chat_container, + "chat", + ), + _scope_profile( + targets, + "triage", + f"deploy/{targets.triage_deployment}", + targets.triage_container, + "triage", + ), + check( + "scopes.surfaces-have-distinct-hosts", + "Chat, agent, and triage are published on distinct hostnames", + "scopes", + "distinct_count", + [ + step( + "hosts", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "ingress", + "-o", + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + ) + ], + {"step": "hosts", "minimum": 3}, + ), + ] + + +def platform_checks(targets: Targets) -> list[CheckSpec]: + """Return the baseline, identity, routing, and scope checks.""" + return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py new file mode 100644 index 00000000..a2e1abea --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the distributed worker pool and the visible surfaces. + +A pool is only distributed if its workers occupy distinct nodes and distinct +volumes and leave state ownership with the coordinator; a replica count alone +satisfies none of that. A surface is only working if the work it did is visible +in it, which is why the session and Kanban checks look at the same data the UI +renders rather than at whether a pod is Running. + +The ephemeral entries are placeholders in a default run. They report NOT_RUN +until the mutation mode is separately armed, and the armed mode replaces them +with real, mandatory results. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CHAT, + INIT_NAME_PROJECTION, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, +) +from hermes_handoff_catalog import step as step +from hermes_handoff_model import EPHEMERAL, CheckSpec + +KANBAN_STATUSES = [ + "archived", + "blocked", + "done", + "ready", + "review", + "running", + "scheduled", + "todo", + "triage", +] + + +def _pool(targets: Targets) -> list[CheckSpec]: + selector = targets.pool_selector + return [ + check( + "pool.three-workers-are-ready", + "The distributed execution pool has its full complement of ready workers", + "pool", + "json_numeric", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": { + "status.readyReplicas": {"min": targets.pool_replicas}, + "spec.replicas": {"min": targets.pool_replicas}, + }, + }, + ), + check( + "pool.workers-occupy-distinct-nodes", + "Each pool worker runs on its own node", + "pool", + "distinct_count", + [ + step( + "nodes", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + selector, + "-o", + NODE_NAME_PROJECTION, + ) + ], + { + "step": "nodes", + "minimum": targets.pool_replicas, + "total_equals": targets.pool_replicas, + }, + rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", + ), + check( + "pool.workers-own-distinct-volumes", + "Each pool worker has its own durable workspace", + "pool", + "distinct_count", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "persistentvolumeclaims", + "--selector", + selector, + "-o", + "name", + ) + ], + {"step": "claims", "minimum": targets.pool_replicas}, + ), + check( + "pool.workers-are-tokenless", + "Pool workers mount no Kubernetes API token", + "pool", + "json_field", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": {"spec.template.spec.automountServiceAccountToken": False}, + }, + rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + ), + check( + "pool.coordinator-retains-sole-state-ownership", + "No pool worker mounts the coordinator's durable state", + "pool", + "names_absent", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "names": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", + ), + *_pool_env(targets), + ] + + +def _pool_env(targets: Targets) -> list[CheckSpec]: + """Return the assignment-safety check only when a site declares what to expect. + + A ``names_present`` assertion over an empty expectation would pass on any + workload at all, which is worse than not asserting: it reads as coverage. + """ + if not targets.pool_worker_env: + return [] + return [ + check( + "pool.assignment-safety-knobs-are-configured", + "Pool workers carry the lease and de-duplication configuration", + "pool", + "names_present", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + ) + ], + {"step": "env", "names": targets.pool_worker_env}, + rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", + ) + ] + + +def _surfaces(targets: Targets) -> list[CheckSpec]: + checks = [ + check( + "surfaces.kanban-activity-is-visible", + "Durable Kanban activity is visible to the agent surface", + "surfaces", + "all_of_field", + [ + step( + "tasks", + SELF, + "hermes", + "kanban", + "list", + "--json", + record=False, + max_bytes=targets.listing_bytes, + ) + ], + {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, + rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", + ), + check( + "surfaces.durable-sessions-render-without-blank-rows", + "Durable sessions are listed with a preview, a source, and an id", + "surfaces", + "lines_match", + [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="A row whose leading column is blank is the session-render regression seen from the data side.", + ), + check( + "surfaces.agent-ui-session-patches-are-deployed", + "The agent surface runs the session-activity rendering patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", + ), + check( + "surfaces.chat-session-patches-are-deployed", + "The chat surface runs the API-session continuity patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.chat_init_containers}, + ), + check( + "surfaces.chat-telegram-topic-state-is-durable", + "Telegram topic and tenant state is held on durable storage", + "surfaces", + "names_present", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.chat_router_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "contains": ("state",)}, + rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", + ), + ] + if targets.expect_telegram_sessions: + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", + ) + ) + if targets.chat_config_revision: + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", + ) + ) + return checks + + +def _ephemeral() -> list[CheckSpec]: + return [ + check( + f"ephemeral.{name}", + title, + "ephemeral", + "not_armed", + [], + {"reason": reason}, + mandatory=False, + scope=EPHEMERAL, + rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", + ) + for name, title, reason in ( + ( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + "ephemeral mutation mode is not armed", + ), + ( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + "ephemeral mutation mode is not armed", + ), + ( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + "ephemeral mutation mode is not armed", + ), + ( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + "ephemeral mutation mode is not armed", + ), + ) + ] + + +def worker_checks(targets: Targets) -> list[CheckSpec]: + """Return the pool, surface, and ephemeral-placeholder checks.""" + return [*_pool(targets), *_surfaces(targets), *_ephemeral()] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py new file mode 100644 index 00000000..5456482b --- /dev/null +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""The separately armed, self-cleaning mutation mode of the acceptance harness. + +Proving that a worker can still open a draft pull request means actually opening +one, which is the only part of this harness that writes anything. It is +therefore a distinct mode with a distinct policy: an explicit confirmation +phrase, a caller-supplied unique ref, a preflight that refuses a protected +target before any network call, and a cleanup whose *verification* is itself a +mandatory check. + +The push target is guarded, not the pull-request base. Opening a pull request +against `main` writes nothing to `main`; pushing to it does. The refusal is +checked by name and by pattern, and the arming self-test exercises it on every +armed run rather than trusting that it was checked once. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass + +from hermes_handoff_exec import Outcome, Runner, Vantage +from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Everything the armed mode needs, and nothing it can infer for itself.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = "main" + + @property + def ref(self) -> str: + """Return the exact, unique branch this run is allowed to create.""" + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + trimmed = name.strip().removeprefix("refs/heads/").strip("/") + return trimmed.lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def preflight(request: ArmRequest, expected_repo: str) -> None: + """Validate an arming request before any network call is made.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if request.repo != expected_repo: + raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + assert_push_target_allowed(request.ref) + if not normalise_branch(request.base): + raise ArmingError("a pull-request base branch is required") + + +def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: + return CheckSpec( + id=f"ephemeral.{identifier}", + title=title, + group="ephemeral", + rule="armed", + steps=(), + mandatory=True, + scope=EPHEMERAL, + rationale=rationale, + ) + + +def _result( + identifier: str, + title: str, + status: str, + reason: str, + outcomes: Sequence[Outcome] = (), + evidence: dict | None = None, + rationale: str = "", +) -> CheckResult: + return CheckResult( + spec=_spec(identifier, title, rationale), + status=status, + reason=reason, + evidence=evidence or {}, + outcomes=list(outcomes), + ) + + +def _protected_refusal_result() -> CheckResult: + """Exercise the protected-branch guard on this run, not on a past one.""" + refused: list[str] = [] + accepted: list[str] = [] + for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): + try: + assert_push_target_allowed(candidate) + except ArmingError: + refused.append(candidate) + else: + accepted.append(candidate) + evidence = {"refused": refused, "accepted": accepted} + if accepted: + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + FAIL, + f"the guard accepted {accepted}", + evidence=evidence, + ) + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + PASS, + "every protected and malformed push target was refused", + evidence=evidence, + ) + + +def _detail(outcome: Outcome) -> str: + """Return the most informative bounded explanation an outcome carries.""" + return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + + +def _pull_request_number(outcome: Outcome) -> int | None: + match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) + return int(match.group(1)) if match else None + + +def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: + """Push a unique ephemeral ref, open a draft pull request, and remove both. + + Every step is checked and every artifact is removed. Cleanup runs even when + an earlier step failed, and the cleanup verification is reported as its own + mandatory check so a half-removed branch cannot pass as success. + """ + api = f"/api/v1/repos/{request.repo}" + results = [_protected_refusal_result()] + outcomes: list[Outcome] = [] + + existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) + outcomes.append(existing) + if not existing.ok: + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + NOT_RUN, + f"could not confirm the ref is unused: {_detail(existing)}", + outcomes, + ) + ) + return results + if existing.stdout.strip(): + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + FAIL, + f"{request.ref} already exists on {request.remote}; pick a fresh token", + outcomes, + ) + ) + return results + + pushed = runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + ) + outcomes.append(pushed) + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if pushed.ok else FAIL, + f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", + [pushed], + {"ref": request.ref}, + ) + ) + + number: int | None = None + if pushed.ok: + created = runner.run( + ( + GITEA_CLIENT, + "POST", + f"{api}/pulls", + "--field", + f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe", + "--field", + f"head={request.ref}", + "--field", + f"base={normalise_branch(request.base)}", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + ), + vantage, + ) + outcomes.append(created) + number = _pull_request_number(created) if created.ok else None + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if created.ok and number else FAIL, + f"opened draft pull request #{number}" + if created.ok and number + else f"draft creation failed: {_detail(created)}", + [created], + {"pull_request": number}, + ) + ) + else: + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + NOT_RUN, + "the ephemeral branch was never pushed", + ) + ) + + results.append(_cleanup(runner, vantage, request, api, number)) + return results + + +def _cleanup( + runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None +) -> CheckResult: + """Close the pull request, delete the ref, and prove both are gone.""" + outcomes: list[Outcome] = [] + problems: list[str] = [] + + if number is not None: + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} could not be closed") + + deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) + outcomes.append(deleted) + + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ran: + problems.append(f"could not verify removal: {_detail(remaining)}") + elif remaining.stdout.strip(): + problems.append(f"{request.ref} still exists on {request.remote}") + + if number is not None: + state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(state) + if not state.ok: + problems.append(f"could not confirm pull request #{number} is closed") + elif '"state":"open"' in state.stdout.replace(" ", ""): + problems.append(f"pull request #{number} is still open") + + if problems: + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL, + "; ".join(problems), + outcomes, + {"ref": request.ref, "pull_request": number}, + ) + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + PASS, + "the ephemeral ref and pull request are gone and their absence was re-checked", + outcomes, + {"ref": request.ref, "pull_request": number}, + ) diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py new file mode 100644 index 00000000..72e3558c --- /dev/null +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Text-shaped evaluators for the handoff acceptance harness. + +These rules cover evidence that arrives as a refusal, an exit status, or a list +of names. The ``denied`` rule is the load-bearing one: it refuses to classify on +an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, +not what the API server does — so a real request must have been made and +refused before it will pass. + +Importing this module also registers the JSON-shaped rules, so a caller only has +to import one place to have the full catalog vocabulary available. +""" + +from __future__ import annotations + +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) +from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec +from hermes_handoff_rules import ( + EVALUATORS, + Evaluation, + evaluate, + evaluator, + lines, + looks_denied, + missing_steps, + step_keys, +) + +__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] + + +@evaluator("denied") +def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when the review says no and a real request was refused.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + attempts = step_keys(spec, ATTEMPT) + if not attempts: + return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + evidence: dict[str, Any] = {} + for key in step_keys(spec, REVIEW): + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + ) + for key in attempts: + outcome = outcomes[key] + if outcome.ok: + return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + if not looks_denied(outcome): + return Evaluation( + NOT_RUN, + f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", + evidence, + ) + evidence[key] = "refused" + return Evaluation(PASS, "review and live attempt were both refused", evidence) + + +@evaluator("allowed") +def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required step succeeded and every review said yes.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + evidence: dict[str, Any] = {} + for step in spec.steps: + outcome = outcomes[step.key] + if step.kind == REVIEW: + verdict = outcome.stdout.strip().lower() + evidence[step.key] = verdict + if verdict != "yes": + return Evaluation( + FAIL, + f"authorization review {step.key} reported {verdict or 'nothing'}", + evidence, + ) + continue + if not outcome.ok: + return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + evidence[step.key] = "succeeded" + return Evaluation(PASS, "required access is present", evidence) + + +def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: + return {name for pattern in patterns for name in observed if pattern in name} + + +def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + return (set(lines(outcome)), key) + + +@evaluator("names_absent") +def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when no forbidden name or substring appears in a step's lines.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + offenders = sorted( + (observed & set(spec.expect.get("names", ()))) + | _matched(observed, tuple(spec.expect.get("contains", ()))) + ) + evidence = {"observed_count": len(observed), "offenders": offenders} + if offenders: + return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation(PASS, "no forbidden names are present", evidence) + + +@evaluator("names_present") +def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required name, and every required substring, appears.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + missing = sorted(set(spec.expect.get("names", ())) - observed) + missing += sorted( + pattern + for pattern in spec.expect.get("contains", ()) + if not any(pattern in name for name in observed) + ) + evidence = {"observed_count": len(observed), "missing": missing} + if missing: + return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + if not observed: + return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) + return Evaluation(PASS, "every required name is present", evidence) + + +@evaluator("stdout_matches") +def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare a step's trimmed stdout against an expected value or set.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + actual = outcome.stdout.strip() + expected = spec.expect["equals"] + allowed = expected if isinstance(expected, list) else [expected] + evidence = {"observed": actual} + if actual not in allowed: + return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation(PASS, "output matches the expected value", evidence) + + +@evaluator("distinct_count") +def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a step yields enough distinct rows — and, optionally, exactly N. + + Distinctness is the point for placement checks: three worker pods pinned to + one node satisfy a replica count and defeat the isolation the replica count + was standing in for. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + values = lines(outcome) + distinct = sorted(set(values)) + minimum = spec.expect["minimum"] + evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + if not values: + return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) + if len(distinct) < minimum: + return Evaluation( + FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + ) + total_equals = spec.expect.get("total_equals") + if total_equals is not None and len(values) != total_equals: + return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) + + +@evaluator("lines_match") +def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every data row of a tabular listing matches a shape. + + Some surfaces only speak tables. A blank leading column is exactly what the + session-render regression looked like, so asserting the shape of each row is + a real check rather than a proxy for one. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + pattern = re.compile(spec.expect["pattern"]) + rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] + rows = [row for row in rows if row.strip()] + offenders = [row[:60] for row in rows if not pattern.search(row)] + evidence = {"rows": len(rows), "offenders": offenders[:8]} + if len(rows) < spec.expect.get("minimum", 1): + return Evaluation( + NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", evidence + ) + if offenders: + return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) + + +@evaluator("vantages_agree") +def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when two independent observations report the same fact. + + A disagreement is a finding in its own right: it usually means the manifest + on record and the workload actually running have drifted apart, and either + one alone would have looked clean. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + left_key, right_key = spec.expect["steps"] + observations: dict[str, list[str]] = {} + for key in (left_key, right_key): + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observations[key] = sorted(lines(outcome)) + evidence = {key: value[:16] for key, value in observations.items()} + if observations[left_key] != observations[right_key]: + return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + if not observations[left_key]: + return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation(PASS, "both vantages report the same state", evidence) + + +@evaluator("not_armed") +def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: + """Report the fixed NOT_RUN a default, read-only run must produce.""" + return Evaluation( + NOT_RUN, + spec.expect.get("reason", "ephemeral mutation mode is not armed"), + {"armed": False}, + ) + + +# Two observations of the same fact agree whether or not they came from +# different vantages; the dual-vantage case is just the important one. +EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py new file mode 100644 index 00000000..200d82e0 --- /dev/null +++ b/scripts/ops/hermes_handoff_exec.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Bounded, screened command execution for the handoff acceptance harness. + +Every probe runs through :class:`Runner`, which owns the three budgets a +long-running acceptance sweep needs: a per-command timeout, a whole-run +deadline, and a per-capture byte budget. Exhausting the deadline does not +silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a +NO_GO, so a slow cluster can never be mistaken for a clean one. + +Subprocesses never inherit the caller's environment. A probe that echoed +`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child +environment is rebuilt from an allowlist of names whose values the harness +already needs to reach the cluster and the forge. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field + +from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text + +DEFAULT_COMMAND_TIMEOUT = 25.0 +DEFAULT_RUN_DEADLINE = 900.0 + +# Names whose values the harness needs to reach the cluster and the forge. The +# values are handed to the child process and are never recorded. +ENV_PASSTHROUGH = ( + "GIT_ASKPASS", + "GITEA_BASE_URL", + "HERMES_HOME", + "HOME", + "JENKINS_BASE_URL", + "KUBECONFIG", + "LANG", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "VAULT_ADDR", +) + +DEADLINE_ERROR = "deadline-exceeded" +TIMEOUT_ERROR = "timeout" + +OPERATOR = "operator" +SELF = "self" + +# Tools that accept a `--context` flag directly after the binary name. +CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") + + +@dataclass(frozen=True) +class Outcome: + """The screened, bounded result of one probe command.""" + + argv: tuple[str, ...] + vantage: str + returncode: int | None = None + stdout: str = "" + stderr: str = "" + truncated: bool = False + duration_ms: int = 0 + error: str | None = None + + @property + def ok(self) -> bool: + """Report whether the command ran to completion and succeeded.""" + return self.error is None and self.returncode == 0 + + @property + def ran(self) -> bool: + """Report whether the command reached the target at all.""" + return self.error is None + + @property + def combined(self) -> str: + """Return stdout and stderr joined, for message matching.""" + return "\n".join(part for part in (self.stdout, self.stderr) if part) + + def as_dict(self) -> dict[str, object]: + """Return a JSON-serialisable view of the outcome.""" + return { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + } + + +@dataclass(frozen=True) +class Vantage: + """A named identity and transport a probe can be observed from. + + ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the + target container, so the inner command authenticates as the pod's own + service account instead of as the operator. ``inserts`` carries per-binary + flags — the operator's `--context`, which must not follow the command into + the pod. + """ + + name: str + prefix: tuple[str, ...] = () + inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + env: Mapping[str, str] = field(default_factory=dict) + description: str = "" + + def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: + """Return ``argv`` addressed to this vantage.""" + command = tuple(argv) + if command: + extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) + if extra: + command = (command[0], *extra, *command[1:]) + return (*self.prefix, *command) + + +def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: + """Return the child environment: allow-listed names plus vantage overrides.""" + origin = os.environ if source is None else source + environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} + environment.setdefault("LC_ALL", "C") + environment.update(vantage.env) + return environment + + +def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: + """Return the external read-only operator vantage.""" + inserts = ( + {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} + ) + detail = " ".join( + part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part + ) + return Vantage( + name=OPERATOR, + inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip(), + ) + + +def pod_vantage( + namespace: str, + pod: str, + container: str, + operator: Vantage | None = None, +) -> Vantage: + """Return the in-pod vantage: the inner command authenticates as the pod.""" + reach = operator.inserts.get("kubectl", ()) if operator else () + return Vantage( + name=SELF, + prefix=( + "kubectl", + *reach, + "exec", + "--namespace", + namespace, + pod, + "--container", + container, + "--", + ), + env=dict(operator.env) if operator else {}, + description=f"in-pod {namespace}/{pod}[{container}]", + ) + + +class Runner: + """Run probe commands under policy, timeout, deadline, and byte budgets.""" + + def __init__( + self, + mode: str = READ_ONLY, + command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + deadline_seconds: float = DEFAULT_RUN_DEADLINE, + max_bytes: int = DEFAULT_MAX_BYTES, + clock: Callable[[], float] = time.monotonic, + spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + environ: Mapping[str, str] | None = None, + ) -> None: + self.mode = mode + self.command_timeout = command_timeout + self.max_bytes = max_bytes + self._clock = clock + self._spawn = spawn + self._environ = environ + self._deadline_at = clock() + deadline_seconds + self.commands_run = 0 + + @property + def remaining_seconds(self) -> float: + """Return the seconds left before the whole-run deadline.""" + return self._deadline_at - self._clock() + + def run( + self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + ) -> Outcome: + """Run one command, returning a screened outcome and never raising.""" + budget_bytes = self.max_bytes if max_bytes is None else max_bytes + addressed = vantage.wrap(argv) + try: + check_argv(addressed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + + budget = min(self.command_timeout, self.remaining_seconds) + if budget <= 0: + return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) + + started = self._clock() + self.commands_run += 1 + try: + completed = self._spawn( + list(addressed), + capture_output=True, + text=True, + timeout=budget, + check=False, + env=build_environment(vantage, self._environ), + ) + except subprocess.TimeoutExpired: + return Outcome( + argv=addressed, + vantage=vantage.name, + error=TIMEOUT_ERROR, + duration_ms=int((self._clock() - started) * 1000), + ) + except OSError as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" + ) + + stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) + stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + return Outcome( + argv=addressed, + vantage=vantage.name, + returncode=completed.returncode, + stdout=stdout, + stderr=stderr, + truncated=cut_out or cut_err, + duration_ms=int((self._clock() - started) * 1000), + ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py new file mode 100644 index 00000000..6c10e52d --- /dev/null +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""JSON-shaped evaluators for the handoff acceptance harness. + +These rules cover the evidence that arrives as structured output — Kubernetes +objects, forge pull requests, provider health files, and the two append-only +evidence logs the platform keeps. They share the fail-closed contract of every +other rule: an absent path, an unparsable payload, or an empty result set is +``NOT_RUN``, never a quiet pass. +""" + +from __future__ import annotations + +import datetime as dt +import json +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec +from hermes_handoff_rules import ( + Evaluation, + dotted, + evaluator, + is_list, + missing_steps, + parsed_step, +) + + +@evaluator("json_field") +def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare dotted JSON paths of one step's output against expected values.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + mismatches: list[str] = [] + for path, expected in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + evidence[path] = actual + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") + if mismatches: + return Evaluation(FAIL, "; ".join(mismatches), evidence) + return Evaluation(PASS, "observed fields match the release contract", evidence) + + +@evaluator("json_numeric") +def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare numeric JSON paths against inclusive minimum/maximum bounds.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + problems: list[str] = [] + for path, bounds in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + if not isinstance(actual, (int, float)) or isinstance(actual, bool): + return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + evidence[path] = actual + if "min" in bounds and actual < bounds["min"]: + problems.append(f"{path}={actual} below minimum {bounds['min']}") + if "max" in bounds and actual > bounds["max"]: + problems.append(f"{path}={actual} above maximum {bounds['max']}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, "numeric fields are within the expected bounds", evidence) + + +@evaluator("json_recent") +def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a timestamp field is fresh enough to be evidence about now. + + Stale provider or routing evidence is worse than none: it describes a state + the release no longer has. A timestamp older than the allowed age is a FAIL, + and one the harness cannot parse is NOT_RUN. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + now = spec.expect["now"] + evidence: dict[str, Any] = {} + stale: list[str] = [] + for path, max_age in spec.expect["fields"].items(): + try: + raw = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + try: + observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) + if observed.tzinfo is None: + observed = observed.replace(tzinfo=dt.timezone.utc) + age = (now - observed).total_seconds() + evidence[path] = {"age_seconds": int(age)} + if age > max_age: + stale.append(f"{path} is {int(age)}s old, limit {max_age}s") + if stale: + return Evaluation(FAIL, "; ".join(stale), evidence) + return Evaluation(PASS, "observed timestamps are fresh", evidence) + + +@evaluator("all_of_field") +def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every selected element of a JSON array matches expected fields.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + elements = dotted(payload, spec.expect.get("array", "")) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"array path is absent from step {key}") + if not is_list(elements): + return Evaluation(NOT_RUN, f"step {key} did not return an array") + key_field = spec.expect.get("key_field", "") + wanted = spec.expect.get("keys") + exempt = set(spec.expect.get("exempt_keys", ())) + seen: list[Any] = [] + problems: list[str] = [] + evidence: dict[str, Any] = {} + for element in elements: + identity = dotted(element, key_field) if key_field else None + if wanted is not None and identity not in wanted: + continue + if identity in exempt: + continue + seen.append(identity) + for path in spec.expect.get("non_empty", ()): + try: + value = dotted(element, path) + except (KeyError, ValueError, IndexError): + value = None + if value in (None, "", [], {}): + problems.append(f"{identity}: {path} is empty") + for path, expected in spec.expect.get("fields", {}).items(): + try: + actual = dotted(element, path) + except (KeyError, ValueError, IndexError): + actual = None + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + evidence["examined"] = seen[:32] + if wanted is not None: + absent = [item for item in wanted if item not in seen] + if absent: + return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + elif not seen: + return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) + if problems: + return Evaluation(FAIL, "; ".join(problems[:8]), evidence) + return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + + +@evaluator("json_record") +def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Capture named JSON paths as evidence; pass when all of them resolve. + + Recording is a release requirement in its own right — the runbook has to + name the exact heads it was verified against — so an unreachable source is a + NOT_RUN rather than an empty section. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + for label, path in spec.expect["record"].items(): + try: + evidence[label] = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation(PASS, "state recorded", evidence) + + + +@evaluator("routing_evidence") +def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Summarise a Switchyard routing log tail into provider/effort coverage. + + Route identifiers are ``///``, so provider, + model family, effort tier, and lane all come from the same field. Fallback + and failure evidence is a separate ``fallback_reason`` on the record that + was retried. The parser tolerates a clipped first line: a tail of a growing + file starts mid-record, and discarding one row is better than a NOT_RUN that + reads as an outage. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + records = [] + for line in outcome.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except ValueError: + continue + routes = [ + str(record.get("model") or "").split("/") + for record in records + if str(record.get("model") or "").count("/") == 3 + ] + fallbacks = [record for record in records if record.get("fallback_reason")] + stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + evidence: dict[str, Any] = { + "records": len(records), + "routed": len(routes), + "lanes": sorted({route[0] for route in routes}), + "providers": sorted({route[1] for route in routes}), + "families": sorted({route[2] for route in routes}), + "efforts": sorted({route[3] for route in routes}), + "fallback_events": len(fallbacks), + "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), + "latest_ts": stamps[-1] if stamps else "", + } + if not records: + return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + missing = [ + f"{label}:{item}" + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ) + for item in wanted + if item not in evidence[f"{label}s"] + ] + if spec.expect.get("require_fallback_evidence") and not fallbacks: + missing.append("fallback:none-observed") + max_age = spec.expect.get("max_age_seconds") + now = spec.expect.get("now") + if max_age and now: + try: + newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) + age = (now - newest).total_seconds() + evidence["age_seconds"] = int(age) + if age > max_age: + missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + if missing: + return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation(PASS, "routing evidence covers the required lanes", evidence) + + +@evaluator("flux_health") +def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Classify a compact Flux projection into suspensions and unhealthy objects. + + Each row is ``namespace/namesuspendType=Status,...``. An object is + unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no + Healthy condition and no in-flight ``Reconciling=True``. That last clause + matters: Flux flips Ready to False for the duration of a reconcile, so + without it the verdict depends on when the snapshot was taken. Suspended + objects are excluded from the health tally and judged against the + expected-suspension list instead, because a deliberate park is a different + fact from a broken reconcile. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + expected = set(spec.expect.get("expected_suspensions", ())) + suspended: list[str] = [] + unhealthy: list[str] = [] + total = 0 + for line in outcome.stdout.splitlines(): + if not line.strip(): + continue + total += 1 + name, _, rest = line.partition("\t") + suspend, _, conditions = rest.partition("\t") + if suspend.strip().lower() == "true": + suspended.append(name.strip()) + continue + states = dict( + pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair + ) + healthy = states.get("Healthy") + reconciling = states.get("Reconciling") == "True" + ready_failed = states.get("Ready") == "False" + if healthy == "False" or (healthy is None and ready_failed and not reconciling): + unhealthy.append(name.strip()) + unexpected = sorted(set(suspended) - expected) + evidence = { + "total": total, + "suspended": sorted(suspended), + "unexpected_suspensions": unexpected, + "unhealthy": sorted(unhealthy), + } + if not total: + return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence) + problems = [] + if unexpected: + problems.append(f"unexpected suspensions: {', '.join(unexpected)}") + if unhealthy: + problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence) diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py new file mode 100644 index 00000000..d34b9ebc --- /dev/null +++ b/scripts/ops/hermes_handoff_model.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Result model, aggregation, and rendering for the handoff acceptance harness. + +The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only +way a check can be excused, and a check can only reach it by matching a scope +rule that a reviewer wrote down; anything the harness merely failed to observe +is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a +``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody +maintained can never round up to a release. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_redaction import contains_credential_shape, scrub + +PASS = "PASS" +FAIL = "FAIL" +NOT_RUN = "NOT_RUN" +NOT_APPLICABLE = "NOT_APPLICABLE" +STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE) + +GO = "GO" +NO_GO = "NO_GO" + +ALWAYS = "always" +EPHEMERAL = "ephemeral" + +HARNESS = "hermes-handoff-acceptance" +SCHEMA_VERSION = 1 + + +REVIEW = "review" +ATTEMPT = "attempt" +READ = "read" +STEP_KINDS = (REVIEW, ATTEMPT, READ) + + +@dataclass(frozen=True) +class Step: + """One command a check needs, and the vantage it must be observed from. + + ``kind`` separates a self access review (``review``) from a real request the + identity is expected to be refused (``attempt``). A deny check needs both: + an authorization review states policy, only an attempt proves the API server + enforces it. + + ``record`` is cleared for bulk evidence — a routing log tail, a large object + listing — which the harness parses but must not paste into the artifact. + ``max_bytes`` raises the capture budget for that same bulk evidence, where a + default-sized truncation would silently narrow the window being examined. + """ + + key: str + vantage: str + argv: tuple[str, ...] + kind: str = READ + optional: bool = False + record: bool = True + max_bytes: int | None = None + + +@dataclass(frozen=True) +class CheckSpec: + """A declarative acceptance check. + + ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the + catalog declarative means a reviewer reads what is asserted, not how it is + plumbed, and the harness can prove structural properties over every check + (no impersonation, no mutation, both vantages present) before running any. + """ + + id: str + title: str + group: str + rule: str + steps: tuple[Step, ...] = () + expect: Mapping[str, Any] = field(default_factory=dict) + mandatory: bool = True + scope: str = ALWAYS + rationale: str = "" + + @property + def vantages(self) -> tuple[str, ...]: + """Return the distinct vantages this check draws evidence from.""" + seen: list[str] = [] + for step in self.steps: + if step.vantage not in seen: + seen.append(step.vantage) + return tuple(seen) + + +@dataclass +class CheckResult: + """The classified outcome of one acceptance check.""" + + spec: CheckSpec + status: str + reason: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + outcomes: list[Outcome] = field(default_factory=list) + + @property + def blocking(self) -> bool: + """Report whether this result alone forces NO_GO.""" + return self.spec.mandatory and self.status in {FAIL, NOT_RUN} + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return a JSON-serialisable, screened view of the result.""" + payload: dict[str, Any] = { + "id": self.spec.id, + "title": self.spec.title, + "group": self.spec.group, + "rule": self.spec.rule, + "scope": self.spec.scope, + "mandatory": self.spec.mandatory, + "vantages": list(self.spec.vantages), + "status": self.status, + "reason": self.reason, + "blocking": self.blocking, + "evidence": self.evidence, + } + if self.spec.rationale: + payload["rationale"] = self.spec.rationale + if include_outcomes: + payload["commands"] = [outcome.as_dict() for outcome in self.outcomes] + return scrub(payload) + + +@dataclass +class VantageRecord: + """What the harness proved about one vantage before trusting its evidence.""" + + name: str + description: str + identity: str = "" + available: bool = False + detail: str = "" + + def as_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable view of the vantage.""" + return scrub( + { + "name": self.name, + "description": self.description, + "identity": self.identity, + "available": self.available, + "detail": self.detail, + } + ) + + +@dataclass +class Report: + """The whole acceptance run: metadata, per-check results, and a verdict.""" + + mode: str + started_at: str + baseline: dict[str, Any] = field(default_factory=dict) + vantages: list[VantageRecord] = field(default_factory=list) + results: list[CheckResult] = field(default_factory=list) + harness_errors: list[str] = field(default_factory=list) + finished_at: str = "" + + @property + def counts(self) -> dict[str, int]: + """Return the number of results in each status.""" + tally = dict.fromkeys(STATUSES, 0) + for result in self.results: + tally[result.status] = tally.get(result.status, 0) + 1 + return tally + + @property + def blocking(self) -> list[CheckResult]: + """Return every result that forces NO_GO, most severe first.""" + order = {FAIL: 0, NOT_RUN: 1} + return sorted( + (result for result in self.results if result.blocking), + key=lambda result: (order.get(result.status, 2), result.spec.id), + ) + + @property + def decision(self) -> str: + """Return GO only when nothing mandatory failed, skipped, or errored.""" + if self.harness_errors or self.blocking: + return NO_GO + return GO + + def group_summary(self) -> dict[str, dict[str, int]]: + """Return per-group status counts, for the human summary.""" + groups: dict[str, dict[str, int]] = {} + for result in self.results: + bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0)) + bucket[result.status] = bucket.get(result.status, 0) + 1 + return groups + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return the machine-readable report.""" + return { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + + def render_summary(self, width: int = 88) -> str: + """Return the concise human summary printed alongside the JSON.""" + counts = self.counts + lines = [ + f"{HARNESS}: {self.decision}", + f" mode={self.mode} started={self.started_at} checks={len(self.results)}", + " " + " ".join(f"{status}={counts[status]}" for status in STATUSES), + ] + for vantage in self.vantages: + state = "available" if vantage.available else "unavailable" + identity = vantage.identity or "unknown" + lines.append(f" vantage {vantage.name}: {state} identity={identity}") + for error in self.harness_errors: + lines.append(f" harness error: {error}") + if self.blocking: + lines.append(" blocking:") + for result in self.blocking: + lines.append(f" {result.status:<15}{result.spec.id}") + lines.append(f" {'':<15}{result.reason[:width]}") + groups = self.group_summary() + lines.append(" groups:") + for name in sorted(groups): + bucket = groups[name] + detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + lines.append(f" {name:<18}{detail}") + return "\n".join(lines) + + +def utc_now(clock: Any = None) -> str: + """Return an ISO-8601 UTC timestamp, injectable for deterministic tests.""" + now = clock() if clock else dt.datetime.now(dt.timezone.utc) + return now.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def unscreened_fields(payload: Any, path: str = "$") -> list[str]: + """Return JSON pointers whose text still looks credential-shaped. + + The report is already screened field by field; this is the invariant check + that runs before anything is written, so a screening gap fails the run + instead of publishing an artifact nobody re-read. + """ + offenders: list[str] = [] + if isinstance(payload, str): + if contains_credential_shape(payload): + offenders.append(path) + elif isinstance(payload, Mapping): + for key, value in payload.items(): + offenders.extend(unscreened_fields(value, f"{path}.{key}")) + elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)): + for index, value in enumerate(payload): + offenders.extend(unscreened_fields(value, f"{path}[{index}]")) + return offenders diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py new file mode 100644 index 00000000..473dd782 --- /dev/null +++ b/scripts/ops/hermes_handoff_policy.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Fail-closed command policy for the Hermes handoff acceptance harness. + +Two properties have to hold structurally, not by convention: + +* the default run mutates nothing, and +* no probe can pull a credential value into the report. + +Both are enforced here, where an argv becomes a subprocess, so a mistake in the +check catalog surfaces as a policy error when the catalog is built rather than +as a mutation or a leak when it runs. `--as` is rejected on the operator side: +impersonation is not evidence about what Hermes can do, and a harness able to +impersonate would be tempted to substitute it for a real self-probe. It stays +available to the inner command of a `kubectl exec`, where an identity is +probing its own refusal to impersonate rather than borrowing authority. + +Shell is reachable only through frozen templates. Free-form `sh -c` cannot be +screened reliably, but a few probes genuinely need field extraction (report that +a password field is locked without ever reading its hash) or an explicit helper +path, so those scripts are named, reviewed here, and parameter-validated instead +of assembled by callers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator + +READ_ONLY = "read-only" +ARMED = "ephemeral-armed" +MODES = (READ_ONLY, ARMED) + +# Rejected everywhere. Impersonation and inline credentials have no honest use +# in an acceptance probe. +FORBIDDEN_ARGS = ( + "--as", + "--as-group", + "--as-uid", + "--token", + "--password", + "--username", + "--client-key", + "--client-certificate", +) +# Reachable only from inside a pod, where an identity is probing its own +# refusal to impersonate rather than borrowing someone else's authority. +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) +GITEA_CLIENT = "/opt/coordinator/gitea_api.py" +ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) + +# Flags that consume the following token, so it is never a subcommand. +VALUE_FLAGS = frozenset( + { + "-c", + "-l", + "-n", + "-o", + "--cluster", + "--container", + "--context", + "--field-selector", + "--kubeconfig", + "--namespace", + "--output", + "--request-timeout", + "--selector", + "--server", + "--since", + "--tail", + "--user", + } +) + +KUBECTL_READ_SUBCOMMANDS = frozenset( + { + "api-resources", + "api-versions", + "auth", + "cluster-info", + "config", + "describe", + "events", + "explain", + "get", + "logs", + "top", + "version", + } +) +# Reachable only through a dry run: the API server authorises the request and +# discards it. A rejection is the evidence, and nothing is persisted either way. +KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( + {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} +) +KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) +KUBECTL_SUBCOMMANDS = ( + KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +) +DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") + +# Resources whose objects carry credential material. Reading them is permitted +# only in name-only output form, so a regression that grants access still cannot +# deliver a secret value into a report. +NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) + +FLUX_READ_SUBCOMMANDS = frozenset( + {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} +) +HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +GIT_READ_SUBCOMMANDS = frozenset( + { + "cat-file", + "config", + "diff", + "fetch", + "for-each-ref", + "log", + "ls-remote", + "ls-tree", + "merge-base", + "remote", + "rev-list", + "rev-parse", + "show", + "status", + } +) +GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) +GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GITEA_READ_METHODS = frozenset({"GET"}) +GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) + +# The Hermes CLI is the only honest source for durable session and Kanban +# activity, so a narrow read-only slice of it is on the allowlist. +HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) + +# Paths that hold credential material. A probe may prove one exists or is +# unreadable; it may never read it outside a frozen template. +SECRET_PATH_RE = re.compile( + r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" + r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +) + +_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") +_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") + +# Frozen shell templates. Each exists because a plain argv cannot express the +# safety property its probe needs. +SHELL_TEMPLATES: dict[str, str] = { + # Report an account's lock state without reading its password hash. + "account_lock_state": ( + 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + ), + # Report the account record's non-secret identity fields. + "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', + # Report supplementary group names the account joins, never their members. + "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", + # Prove a path's presence and mode without reading its bytes. + "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', + # Prove whether a path is readable by this identity, without printing it. + "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', + # List environment variable names only. Values never leave the container. + "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", + # Report a JSON document's top-level key names without any of its values. + "json_keys": ( + 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' + ), + # Report allow-listed non-secret status fields from a provider health file. + "json_fields": ( + 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + ), + # Count non-empty lines, for authorized-key inventories. + "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", + # Tail an append-only evidence log. The harness parses it; the container is + # not assumed to carry an interpreter. + "tail_lines": "tail -n {limit} {path}", + # Prove the worker can still read the repository. `kubectl exec` does not + # inherit the coordinator process environment, so the askpass helper is + # named explicitly rather than assumed; the helper reads runtime state and + # the credential never reaches this command line. + "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", +} + + +class PolicyError(ValueError): + """Raised when a requested command falls outside the safe boundary.""" + + +def template_parameters(template_name: str) -> set[str]: + """Return the parameter names a frozen template requires.""" + template = SHELL_TEMPLATES.get(template_name) + if template is None: + raise PolicyError(f"unknown shell template: {template_name}") + return set(_PLACEHOLDER_RE.findall(template)) + + +def render_shell(template_name: str, **parameters: str) -> str: + """Return a frozen shell template with validated parameters substituted.""" + required = template_parameters(template_name) + template = SHELL_TEMPLATES[template_name] + if set(parameters) != required: + raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + for key, value in parameters.items(): + if not _PARAMETER_RE.fullmatch(value): + raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") + return template.format(**parameters) + + +_RENDERED: set[str] = set() + + +def shell(template_name: str, **parameters: str) -> tuple[str, ...]: + """Return a validated ``sh -c`` argv built from a frozen template.""" + script = render_shell(template_name, **parameters) + _RENDERED.add(script) + return ("sh", "-c", script) + + +def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: + """Yield positional arguments, skipping flags and the values they consume.""" + index = start + while index < len(argv): + item = argv[index] + if item == "--": + return + if item.startswith("-"): + if item in VALUE_FLAGS: + index += 2 + continue + index += 1 + continue + yield item + index += 1 + + +def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: + permitted = IMPERSONATION_ARGS if allow_impersonation else () + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + for forbidden in FORBIDDEN_ARGS: + if forbidden in permitted: + continue + if argument == forbidden or argument.startswith(f"{forbidden}="): + raise PolicyError(f"forbidden argument: {forbidden}") + + +def _reject_secret_paths(argv: tuple[str, ...]) -> None: + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + if SECRET_PATH_RE.search(argument): + raise PolicyError("probes may not name a credential path outside a frozen template") + + +def _assert_name_only_output(argv: tuple[str, ...]) -> None: + if "--output=name" in argv or "-oname" in argv: + return + for index, item in enumerate(argv[:-1]): + if item in {"-o", "--output"} and argv[index + 1] == "name": + return + raise PolicyError("credential-bearing resources may only be read with -o name") + + +def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: + words = list(positionals(argv)) + subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) + if subcommand is None: + raise PolicyError("kubectl subcommand is outside the read-only boundary") + if subcommand != words[0]: + raise PolicyError("kubectl subcommand must precede its resource arguments") + resource = words[1].split(".")[0] if len(words) > 1 else "" + if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: + _assert_name_only_output(argv) + if subcommand in KUBECTL_READ_SUBCOMMANDS: + return + if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if "--" not in argv: + raise PolicyError("kubectl exec requires an explicit -- command boundary") + inner = argv[argv.index("--") + 1 :] + if not inner: + raise PolicyError("kubectl exec requires an inner command") + check_argv(inner, mode, allow_impersonation=True) + return + if any(flag in argv for flag in DRY_RUN_FLAGS): + return + raise PolicyError(f"kubectl {subcommand} requires a dry run") + + +def _first_positional(binary: str, argv: tuple[str, ...]) -> str: + word = next(positionals(argv), None) + if word is None: + raise PolicyError(f"{binary} requires a subcommand") + return word + + +def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: + subcommand = _first_positional(binary, argv) + if subcommand not in allowed: + raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") + + +def _check_git(argv: tuple[str, ...], mode: str) -> None: + subcommand = _first_positional("git", argv) + if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): + raise PolicyError("git config may only read") + if subcommand in GIT_READ_SUBCOMMANDS: + return + if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if GIT_FORCE_FLAGS & set(argv): + raise PolicyError("git push may not force or fan out across refs") + return + raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") + + +def _check_hermes(argv: tuple[str, ...]) -> None: + words = tuple(positionals(argv)) + if not words: + raise PolicyError("hermes requires a subcommand") + if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: + raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + + +def _check_gitea(argv: tuple[str, ...], mode: str) -> None: + if len(argv) < 3: + raise PolicyError("gitea_api.py requires a method and a path") + method = argv[1].upper() + if method in GITEA_READ_METHODS: + return + if method in GITEA_ARMED_METHODS and mode == ARMED: + return + raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") + + +def _check_shell(argv: tuple[str, ...]) -> None: + if len(argv) != 3 or argv[1] != "-c": + raise PolicyError("sh is only reachable as sh -c ") + if argv[2] not in _RENDERED: + raise PolicyError("sh -c script is not a frozen template rendering") + + +def check_argv( + argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False +) -> None: + """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. + + ``allow_impersonation`` is set only for the inner command of a + ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; + the operator impersonating Hermes is not, so `--as` stays unreachable from + the outer, operator-side argv. + """ + if mode not in MODES: + raise PolicyError(f"unknown mode: {mode}") + if not argv: + raise PolicyError("empty command") + binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + if binary in FORBIDDEN_BINARIES: + raise PolicyError(f"forbidden binary: {binary}") + if binary not in ALLOWED_BINARIES: + raise PolicyError(f"binary is not on the harness allowlist: {binary}") + _reject_forbidden_arguments(argv, allow_impersonation) + if binary == "sh": + _check_shell(argv) + return + _reject_secret_paths(argv) + if binary == "kubectl": + _check_kubectl(argv, mode, allow_impersonation) + elif binary == "flux": + _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) + elif binary == "helm": + _check_read_only_tool("helm", argv, HELM_READ_SUBCOMMANDS) + elif binary == "git": + _check_git(argv, mode) + elif binary == "hermes": + _check_hermes(argv) + else: + _check_gitea(argv, mode) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py new file mode 100644 index 00000000..f48344a5 --- /dev/null +++ b/scripts/ops/hermes_handoff_redaction.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Credential redaction and output bounding for the handoff acceptance harness. + +Every string the harness records — command output, failure reasons, evidence +values — passes through :func:`redact` before it reaches a report. Screening is +a fail-closed accident barrier, not proof that text is secret-free: a short +secret under an innocuous key is not reliably distinguishable from prose. The +harness therefore also never asks for a secret in the first place. Probes read +names, shapes, and status fields; they never read environment values, token +files, private keys, cookies, or Vault responses. + +Bounding is part of the same barrier. An unbounded capture is how a debug dump +or a stack trace carrying an authorization header ends up in an artifact, so +captures are truncated at a byte budget before they are screened. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[redacted]" +DEFAULT_MAX_BYTES = 64 * 1024 +TRUNCATION_NOTE = "[truncated]" + +# Keys whose value is never safe to record, matched against a normalised +# (lowercased, separator-stripped) form of the key. +SENSITIVE_KEY_SUFFIXES = ( + "accesskey", + "accesskeyid", + "apikey", + "authtoken", + "clientsecret", + "connectionstring", + "cookie", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "signature", + "signingkey", + "token", + "webhook", +) +SENSITIVE_KEY_EXACT = frozenset( + { + "auth", + "authorization", + "bearer", + "pat", + "pw", + "sas", + "setcookie", + } +) +# A bare `key:` is a structural field name across Kubernetes and TOML far more +# often than it is a credential, so it is screened by value shape and by the +# token patterns rather than by name. The compound forms above still match. + +# Values that cannot hold a credential: booleans, numbers, and anything too +# short to be one. Screening them corrupts structured evidence — a redacted +# `automountServiceAccountToken` is exactly the fact a reader needs — without +# protecting anything. +MIN_SECRET_LENGTH = 8 +NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") + +# Ordered: structural matches first so a nested token is not partially rewritten +# by a looser rule. +_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", + re.DOTALL, + ), + re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), + re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), + re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"(?["']?) + (?P[A-Za-z][A-Za-z0-9_.\ -]{0,63}) + (?P=quote) + (?P\s{0,8}[:=]\s{0,8}) + (?P"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096}) + """ +) + +# Long, mixed-class runs. Deliberately conservative: prose and hex digests of +# fewer than 44 characters are left alone so commit SHAs stay readable. +_ENTROPY_RE = re.compile(r"(? str: + """Return a key lowercased with separators removed, for suffix matching.""" + return _KEY_NOISE_RE.sub("", key.lower()) + + +def is_sensitive_key(key: str) -> bool: + """Report whether a key name means its value must never be recorded.""" + compact = normalise_key(key) + if not compact: + return False + if compact in SENSITIVE_KEY_EXACT: + return True + return compact.endswith(SENSITIVE_KEY_SUFFIXES) + + +def _is_high_entropy(candidate: str) -> bool: + """Report whether a long run mixes enough character classes to be a token.""" + classes = sum( + ( + any(character.islower() for character in candidate), + any(character.isupper() for character in candidate), + any(character.isdigit() for character in candidate), + any(character in "+/_=-" for character in candidate), + ) + ) + return classes >= 3 + + +def could_hold_secret(value: str) -> bool: + """Report whether a raw value is long and opaque enough to be a credential.""" + stripped = value.strip().strip("\"'") + if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH: + return False + try: + float(stripped) + except ValueError: + return True + return False + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + return match.group(0) + # Preserve the original quoting so screened JSON stays parsable JSON. + quoted = value.strip()[:1] in {'"', "'"} + replacement = f'"{REDACTED}"' if quoted else REDACTED + key_quote = match.group("quote") + return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + + +def _redact_entropy(match: re.Match[str]) -> str: + return REDACTED if _is_high_entropy(match.group(0)) else match.group(0) + + +def redact(text: str) -> str: + """Return ``text`` with credential-shaped substrings replaced.""" + if not text: + return text + for pattern in _PATTERNS: + text = pattern.sub(REDACTED, text) + text = _ASSIGNMENT_RE.sub(_redact_assignment, text) + return _ENTROPY_RE.sub(_redact_entropy, text) + + +def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut.""" + if max_bytes <= 0: + return ("", bool(text)) + encoded = text.encode("utf-8", "replace") + if len(encoded) <= max_bytes: + return (text, False) + kept = encoded[:max_bytes].decode("utf-8", "ignore") + return (f"{kept}\n{TRUNCATION_NOTE}", True) + + +def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Bound then redact, in that order, so a huge dump cannot outrun screening.""" + bounded, truncated = bound(text, max_bytes) + return (redact(bounded), truncated) + + +def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: + """Recursively bound and redact a JSON-shaped value. + + Mapping keys are preserved (names are evidence); values under a sensitive + key are dropped outright rather than pattern-screened, because a short + secret would survive screening. + """ + if isinstance(value, str): + return safe_text(value, max_bytes)[0] + if isinstance(value, dict): + return { + str(key): REDACTED + if is_sensitive_key(str(key)) and could_hold_secret(str(item)) + else scrub(item, max_bytes) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [scrub(item, max_bytes) for item in value] + return value + + +def contains_credential_shape(text: str) -> bool: + """Report whether screening would rewrite ``text``. + + Used by the harness self-test to prove a rendered report is clean before it + is written, and by the unit suite as an assertion helper. + """ + return redact(text) != text diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py new file mode 100644 index 00000000..cf9b5b97 --- /dev/null +++ b/scripts/ops/hermes_handoff_rules.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Shared primitives and dispatch for handoff acceptance evaluators. + +Evaluators are registered by name so the check catalog stays declarative: an +entry says *what* it asserts, not how to plumb it. The dispatch here is the one +place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must +never be indistinguishable from a passing release. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import NOT_RUN, CheckSpec + +DENIAL_MARKERS = ( + "forbidden", + "is not allowed", + "unauthorized", + "permission denied", + "cannot get", + "cannot list", + "cannot create", + "cannot delete", + "cannot patch", + "not permitted", + "no such route", + "404 not found", + "403", +) + + +@dataclass +class Evaluation: + """A classification plus the small facts that justify it.""" + + status: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation] +EVALUATORS: dict[str, Evaluator] = {} + + +def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: + """Register an evaluator under the rule name catalog entries use.""" + + def register(function: Evaluator) -> Evaluator: + EVALUATORS[name] = function + return function + + return register + + +def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: + """Return NOT_RUN when a required step never produced usable evidence.""" + for step in spec.steps: + if step.optional: + continue + outcome = outcomes.get(step.key) + if outcome is None: + return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if not outcome.ran: + return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") + return None + + +def looks_denied(outcome: Outcome) -> bool: + """Report whether a real request was refused by policy rather than failing.""" + if outcome.ok: + return False + haystack = outcome.combined.lower() + return any(marker in haystack for marker in DENIAL_MARKERS) + + +def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]: + """Return the keys of every step of one kind.""" + return tuple(step.key for step in spec.steps if step.kind == kind) + + +def is_list(value: Any) -> bool: + """Report whether a value is a JSON array rather than a string.""" + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def dotted(payload: Any, path: str) -> Any: + """Return a value addressed by a dotted path. + + ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list, + which is how the catalog records one field from every pull request or every + Flux object without a bespoke parser per check. + """ + current = payload + segments = [segment for segment in path.split(".") if segment] + for position, raw in enumerate(segments): + name, bracket, indexes = raw.partition("[") + if name: + if not isinstance(current, Mapping) or name not in current: + raise KeyError(path) + current = current[name] + if not bracket: + continue + for chunk in indexes.rstrip("]").split("]["): + if not is_list(current): + raise KeyError(path) + if chunk == "": + remainder = ".".join(segments[position + 1 :]) + return [dotted(item, remainder) if remainder else item for item in current] + offset = int(chunk) + if offset >= len(current): + raise KeyError(path) + current = current[offset] + return current + + +def parse_json(outcome: Outcome) -> Any: + """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. + + Control characters are tolerated. Task bodies and session titles routinely + carry raw newlines and tabs, and refusing to read a report because a human + pasted a tab into a Kanban card would be a false NO_GO. + """ + text = outcome.stdout.strip() + if not text: + raise ValueError("empty output") + return json.loads(text, strict=False) + + +def lines(outcome: Outcome) -> list[str]: + """Return non-empty, stripped stdout lines.""" + return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] + + +def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: + """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + try: + return (parse_json(outcome), key) + except ValueError as exc: + return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}") + + +def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Dispatch to the named evaluator, converting any surprise into NOT_RUN.""" + rule = EVALUATORS.get(spec.rule) + if rule is None: + return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + try: + return rule(spec, outcomes) + except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS + return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py new file mode 100644 index 00000000..cdfaf3c2 --- /dev/null +++ b/scripts/ops/hermes_handoff_run.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Orchestration for the Hermes full-handoff acceptance harness. + +Three things happen before any check runs, and each of them can stop the run on +its own: + +* the catalog is validated structurally — every rule exists, every deny check + carries a real attempt, no operator-side step can impersonate; +* each vantage is resolved and asked who it is; and +* the operator and in-pod identities are compared. If they are the same + principal there is only one vantage, the dual-vantage evidence would be a + restatement rather than a corroboration, and the run is a NO_GO. + +Everything after that is uniform: run a check's steps against its vantage, hand +the outcomes to the named evaluator, and record the classification. A vantage +that could not be resolved yields ``NOT_RUN`` for every check that needs it, +which is exactly the fail-closed answer. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence + +from hermes_handoff_catalog import ( + CHAT, + NODE, + OPERATOR, + SELF, + SWITCHYARD, + VANTAGE_NAMES, + Targets, +) +from hermes_handoff_checks_access import access_checks +from hermes_handoff_checks_delivery import delivery_checks +from hermes_handoff_checks_platform import platform_checks +from hermes_handoff_checks_workers import worker_checks +from hermes_handoff_evaluators import EVALUATORS, evaluate +from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage +from hermes_handoff_model import ( + ATTEMPT, + EPHEMERAL, + NOT_RUN, + STEP_KINDS, + CheckResult, + CheckSpec, + Report, + VantageRecord, + utc_now, +) +from hermes_handoff_policy import IMPERSONATION_ARGS + +UNRECORDED = "[output not recorded]" + + +def build_catalog(targets: Targets) -> list[CheckSpec]: + """Return the whole acceptance catalog for a set of targets.""" + return [ + *platform_checks(targets), + *access_checks(targets), + *delivery_checks(targets), + *worker_checks(targets), + ] + + +def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: + """Return the structural problems that make a catalog unsafe to run.""" + problems: list[str] = [] + seen: set[str] = set() + for spec in specs: + if spec.id in seen: + problems.append(f"{spec.id}: duplicate check id") + seen.add(spec.id) + if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: + problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + for step in spec.steps: + if step.vantage not in VANTAGE_NAMES: + problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + if step.kind not in STEP_KINDS: + problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): + problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") + if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): + problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + return problems + + +def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: + """Return the authenticated username of a vantage and any failure detail.""" + outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + try: + payload = json.loads(outcome.stdout) + except ValueError as exc: + return ("", f"unparsable identity: {exc}") + username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") + if not isinstance(username, str) or not username: + return ("", "identity response carried no username") + return (username, "") + + +def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: + outcome = runner.run( + ( + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + selector, + "--field-selector", + "status.phase=Running", + "-o", + "name", + ), + operator, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + if not names: + return ("", f"no running pod matches {selector}") + return (sorted(names)[0], "") + + +def resolve_vantages( + runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None +) -> tuple[dict[str, Vantage], list[VantageRecord]]: + """Resolve every vantage the catalog names, recording what each one is.""" + operator = operator_vantage(kubeconfig, context) + username, detail = resolve_identity(runner, operator) + records = [ + VantageRecord( + name=OPERATOR, + description=operator.description or "external read-only operator", + identity=username, + available=bool(username), + detail=detail, + ) + ] + vantages: dict[str, Vantage] = {OPERATOR: operator} + if not username: + return (vantages, records) + + for name, selector, container, description in ( + (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SWITCHYARD, + f"app={targets.switchyard_deployment}", + targets.switchyard_container, + "the Switchyard routing evidence log", + ), + (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ): + pod, failure = _first_pod(runner, operator, targets.namespace, selector) + if not pod: + records.append( + VantageRecord(name=name, description=description, available=False, detail=failure) + ) + continue + vantage = pod_vantage(targets.namespace, pod, container, operator) + identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + records.append( + VantageRecord( + name=name, + description=f"{description} ({targets.namespace}/{pod})", + identity=identity, + available=True, + detail=identity_detail, + ) + ) + vantages[name] = vantage + + chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" + present = runner.run( + ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + operator, + ) + if present.ok and present.stdout.strip(): + vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=True, + ) + ) + else: + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=False, + detail=present.error or present.combined[:200] or "chat tenant pod not found", + ) + ) + return (vantages, records) + + +def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: + """Return the identity problems that invalidate dual-vantage evidence.""" + by_name = {record.name: record for record in records} + operator = by_name.get(OPERATOR) + hermes = by_name.get(SELF) + problems: list[str] = [] + if operator is None or not operator.available: + problems.append("the operator vantage is unavailable; no evidence can be collected") + return problems + if hermes is None or not hermes.available: + problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + return problems + if not hermes.identity: + problems.append("the in-pod vantage did not report an identity") + elif operator.identity == hermes.identity: + problems.append( + "the operator and in-pod vantages authenticate as the same principal " + f"({operator.identity}); run the harness with a separate operator kubeconfig" + ) + return problems + + +def run_check( + runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] +) -> CheckResult: + """Run one check's steps and classify the result.""" + outcomes: dict[str, Outcome] = {} + recorded: list[Outcome] = [] + for step in spec.steps: + vantage = vantages.get(step.vantage) + if vantage is None: + outcome = Outcome( + argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + ) + else: + outcome = runner.run(step.argv, vantage, step.max_bytes) + outcomes[step.key] = outcome + recorded.append( + outcome + if step.record + else Outcome( + argv=outcome.argv, + vantage=outcome.vantage, + returncode=outcome.returncode, + stdout=UNRECORDED if outcome.stdout else "", + stderr=UNRECORDED if outcome.stderr else "", + truncated=outcome.truncated, + duration_ms=outcome.duration_ms, + error=outcome.error, + ) + ) + evaluation = evaluate(spec, outcomes) + return CheckResult( + spec=spec, + status=evaluation.status, + reason=evaluation.reason, + evidence=evaluation.evidence, + outcomes=recorded, + ) + + +def run_catalog( + runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] +) -> list[CheckResult]: + """Run every check in the catalog. + + Nothing is skipped once the deadline passes: the runner returns a + deadline outcome for each remaining command, which classifies as + ``NOT_RUN``, so a truncated sweep is visible in the report rather than + silently shorter. + """ + return [run_check(runner, spec, vantages) for spec in specs] + + +def build_report( + runner: Runner, + targets: Targets, + specs: Sequence[CheckSpec], + vantages: Mapping[str, Vantage], + records: Sequence[VantageRecord], + mode: str, + started_at: str, + extra_results: Sequence[CheckResult] = (), +) -> Report: + """Run the catalog and assemble the report, including structural problems.""" + problems = validate_catalog(specs) + vantage_problems(records) + report = Report( + mode=mode, + started_at=started_at, + baseline={ + "namespace": targets.namespace, + "baseline_commit": targets.baseline_commit, + "dependency_pull_requests": list(targets.dependency_pull_requests), + "expected_suspensions": list(targets.expected_suspensions), + "node_count": targets.node_count, + "pool_replicas": targets.pool_replicas, + }, + vantages=list(records), + harness_errors=problems, + ) + armed_ids = {result.spec.id for result in extra_results} + report.results = [ + *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), + *extra_results, + ] + report.results.sort(key=lambda result: result.spec.id) + if runner.remaining_seconds <= 0: + report.harness_errors.append("the run deadline expired before every check completed") + report.finished_at = utc_now() + return report + + +def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: + """Return a NO_GO report for a run that could not start at all.""" + return Report( + mode=mode, + started_at=started_at, + finished_at=utc_now(), + baseline={"namespace": targets.namespace}, + harness_errors=[reason], + results=[ + CheckResult( + spec=CheckSpec( + id="harness.startup", + title="The harness could start and resolve its vantages", + group="harness", + rule="allowed", + ), + status=NOT_RUN, + reason=reason, + ) + ], + ) diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py new file mode 100644 index 00000000..87c8adbe --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -0,0 +1,395 @@ +"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module + +acceptance = load_handoff_module("hermes_handoff_acceptance") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +harness_run = load_handoff_module("hermes_handoff_run") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +class ScriptedCluster: + """A ``subprocess.run`` stand-in that answers by command shape. + + Real runs issue a couple of hundred commands; scripting them positionally + would make every test a brittle ordering puzzle. Matching on the command + line keeps each test about the behaviour it is checking. + """ + + def __init__(self, rules: list[tuple[str, object]], default=None) -> None: + self.rules = rules + self.default = default or completed(stderr="not found", returncode=1) + self.calls: list[list[str]] = [] + + def __call__(self, argv, **_kwargs): + self.calls.append(list(argv)) + line = " ".join(argv) + for fragment, response in self.rules: + if fragment in line: + return response + return self.default + + +def whoami(username: str): + return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + + +HEALTHY_CLUSTER = [ + ("auth whoami", whoami("kubernetes-admin")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), +] + + +def run_cli(monkeypatch, argv, rules): + cluster = ScriptedCluster(rules) + clock = FakeClock() + + def build_runner(**kwargs): + return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + + monkeypatch.setattr(acceptance, "Runner", build_runner) + return (acceptance.main(argv), cluster) + + +def read_report(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_the_parser_exposes_the_documented_controls() -> None: + arguments = acceptance.build_parser().parse_args( + [ + "--namespace", + "hermes", + "--context", + "atlas-operator", + "--dependency-pr", + "14", + "--dependency-pr", + "15", + "--expected-suspension", + "flux-system/parked", + "--node-count", + "21", + "--pool-worker-env", + "HERMES_POOL_LEASE_SECONDS", + "--expect-telegram-sessions", + ] + ) + + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == (14, 15) + assert targets.expected_suspensions == ("flux-system/parked",) + assert targets.node_count == 21 + assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) + assert targets.expect_telegram_sessions is True + + +def test_defaults_fall_back_to_the_shipped_targets() -> None: + arguments = acceptance.build_parser().parse_args([]) + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests + assert targets.expected_suspensions == () + assert targets.repo == catalog.Targets.repo + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["mode"] == policy.READ_ONLY + assert not any("push" in call for call in cluster.calls) + assert not any("POST" in call for call in cluster.calls) + assert not any("DELETE" in call for call in cluster.calls) + writes = [ + call + for call in cluster.calls + if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + ] + assert writes + assert all("--dry-run=server" in call for call in writes) + + +def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + ephemeral = { + check["id"]: check + for check in read_report(output)["checks"] + if check["group"] == "ephemeral" + } + + assert len(ephemeral) == 4 + for check in ephemeral.values(): + assert check["status"] == model.NOT_RUN + assert check["mandatory"] is False + assert check["scope"] == model.EPHEMERAL + + +def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( + monkeypatch, tmp_path +) -> None: + """Nothing observed must never read as nothing wrong.""" + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [("auth whoami", whoami("kubernetes-admin"))], + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["decision"] == model.NO_GO + assert report["counts"][model.PASS] == 0 + assert sum(report["counts"].values()) == len(report["checks"]) + assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + + +def test_partial_failure_is_reported_per_check_rather_than_aborting( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + *HEALTHY_CLUSTER, + ("auth can-i", completed(stdout="no\n", returncode=1)), + ("get namespaces", completed(stdout="namespace/hermes\n")), + ("merge-base", completed()), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + statuses = {check["status"] for check in report["checks"]} + + assert status == acceptance.EXIT_NO_GO + assert model.PASS in statuses and model.NOT_RUN in statuses + assert report["blocking"] + + +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + rules = [ + ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert any("same principal" in error for error in report["harness_errors"]) + + +def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["counts"][model.NOT_RUN] > 0 + assert any("deadline expired" in error for error in report["harness_errors"]) + + +def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + "please", + "--ephemeral-token", + "acceptance-20260817a", + ], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in report["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "main", + ], + HEALTHY_CLUSTER, + ) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in read_report(output)["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), + ("git push", completed()), + ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ("PATCH", completed()), + ("DELETE", completed()), + ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + *HEALTHY_CLUSTER, + ] + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ], + rules, + ) + report = read_report(output) + ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} + + assert report["mode"] == policy.ARMED + assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven + assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS + assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS + assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS + assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS + + pushes = [call for call in cluster.calls if "push" in call] + assert len(pushes) == 1 + assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") + assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator + assert any("DELETE" in call for call in cluster.calls) + + +def test_the_report_is_written_to_stdout_when_no_output_path_is_given( + monkeypatch, capsys +) -> None: + status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) + payload = json.loads(capsys.readouterr().out) + + assert status == acceptance.EXIT_NO_GO + assert payload["harness"] == model.HARNESS + + +def test_the_human_summary_accompanies_the_json_unless_suppressed( + monkeypatch, capsys, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) + captured = capsys.readouterr() + + assert model.HARNESS in captured.err + assert "vantage operator" in captured.err + assert captured.out == "" + + +def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: + """The last barrier: never write an artifact nobody re-read.""" + output = tmp_path / "report.json" + arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") + report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] + + monkey = report.as_dict + report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] + **monkey(), + "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + } + offenders = acceptance.emit(report, arguments) + payload = read_report(output) + + assert offenders == ["$.leak"] + assert payload["decision"] == model.NO_GO + assert "leak" not in payload + assert "failed its own credential screening" in payload["harness_errors"][0] + + +def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: + """The exit status must follow the refusal to publish, not the check tally.""" + output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") + run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [*HEALTHY_CLUSTER, ("exec", leaky)], + ) + + assert model.unscreened_fields(read_report(output)) == [] + assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("flag", ["--help"]) +def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: + with pytest.raises(SystemExit): + acceptance.main([flag]) + + assert "Read-only by default" in capsys.readouterr().out + + +def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: + """Exercise the success exit path without pretending the cluster is clean.""" + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_GO + assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py new file mode 100644 index 00000000..8f5e2de6 --- /dev/null +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -0,0 +1,194 @@ +"""Structural contracts for the handoff acceptance catalog.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +catalog = load_handoff_module("hermes_handoff_catalog") +harness_run = load_handoff_module("hermes_handoff_run") +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +exec_module = load_handoff_module("hermes_handoff_exec") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) +SPECS = harness_run.build_catalog(TARGETS) +BY_ID = {check.id: check for check in SPECS} + +VANTAGES = { + catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), + catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), + catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), + catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), +} + + +def test_the_catalog_is_structurally_valid() -> None: + assert harness_run.validate_catalog(SPECS) == [] + assert len(SPECS) >= 60 + + +def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None: + """A catalog entry that cannot run under policy is a bug found before a run.""" + for check in SPECS: + for step in check.steps: + addressed = VANTAGES[step.vantage].wrap(step.argv) + policy.check_argv(addressed, policy.READ_ONLY) + + +def test_no_step_mutates_outside_a_server_dry_run() -> None: + """`auth can-i create ...` is a read; the subcommand is what decides.""" + for check in SPECS: + for step in check.steps: + subcommand = next(policy.positionals(step.argv), "") + if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: + assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + + +def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: + operator_steps = [ + step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + ] + assert operator_steps + assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + + impersonation = BY_ID["access.impersonation-is-denied"] + attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] + assert any("--as" in step.argv for step in attempts) + + +def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: + deny_checks = [check for check in SPECS if check.rule == "denied"] + assert len(deny_checks) >= 6 + for check in deny_checks: + assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + + +def test_both_vantages_are_represented_across_the_catalog() -> None: + used = {step.vantage for check in SPECS for step in check.steps} + assert catalog.OPERATOR in used and catalog.SELF in used + assert used <= set(catalog.VANTAGE_NAMES) + + +def test_every_rule_named_by_a_check_is_registered() -> None: + for check in SPECS: + if check.scope == model.EPHEMERAL: + assert check.rule == "not_armed" + continue + assert check.rule in evaluators.EVALUATORS, check.id + + +def test_every_evidence_step_referenced_by_expect_exists() -> None: + for check in SPECS: + keys = {step.key for step in check.steps} + named = [check.expect.get("step")] + list(check.expect.get("steps", ())) + for key in filter(None, named): + assert key in keys, f"{check.id} names missing step {key}" + + +def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None: + ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL] + assert len(ephemeral) == 4 + for check in ephemeral: + assert check.mandatory is False + assert evaluators.evaluate(check, {}).status == model.NOT_RUN + + +def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: + for number in TARGETS.dependency_pull_requests: + check = BY_ID[f"baseline.dependency-pr-{number}-merged"] + assert check.mandatory + assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + + +def test_the_baseline_commit_gate_names_the_configured_commit() -> None: + check = BY_ID["baseline.origin-main-descends-merged-work"] + assert TARGETS.baseline_commit in check.steps[0].argv + assert "--is-ancestor" in check.steps[0].argv + + +def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: + """A names_present over an empty expectation would pass on any workload.""" + assert "pool.assignment-safety-knobs-are-configured" not in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID + + configured = harness_run.build_catalog( + catalog.Targets( + now=NOW, + pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), + chat_config_revision="20260816-telegram-topics", + expect_telegram_sessions=True, + ) + ) + identifiers = {check.id for check in configured} + assert "pool.assignment-safety-knobs-are-configured" in identifiers + assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers + assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers + assert harness_run.validate_catalog(configured) == [] + + +def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.steps[0].record is False + assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes + + kanban = BY_ID["surfaces.kanban-activity-is-visible"] + assert kanban.steps[0].record is False + + +def test_freshness_gates_carry_the_run_clock() -> None: + for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + assert BY_ID[identifier].expect["now"] == NOW + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.expect["now"] == NOW + assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds + + +def test_check_identifiers_are_unique_and_grouped() -> None: + identifiers = [check.id for check in SPECS] + assert len(identifiers) == len(set(identifiers)) + for check in SPECS: + assert check.id.split(".")[0] + assert check.group + + +@pytest.mark.parametrize( + "identifier", + [ + "identity.no-provider-api-key-in-pod", + "identity.no-provider-api-key-in-manifest", + "forge.worker-holds-no-git-credential-variable", + "access.secrets-are-denied", + "access.service-account-tokens-are-denied", + "access.workload-mutation-is-denied", + "access.pod-exec-is-denied", + "access.required-reads-are-allowed", + "access.pod-logs-are-allowed", + "access.flux-and-helm-status-are-allowed", + "access.namespace-view-agrees-across-vantages", + "nodes.account-password-is-locked", + "nodes.hardening-covers-every-node", + "build.builder-service-account-is-tokenless", + "build.harbor-immutability-rule-is-applied", + "build.deployed-image-is-digest-pinned", + "reliability.finalization-and-replay-patches-are-deployed", + "pool.three-workers-are-ready", + "pool.workers-occupy-distinct-nodes", + "pool.workers-own-distinct-volumes", + "pool.coordinator-retains-sole-state-ownership", + "surfaces.kanban-activity-is-visible", + "surfaces.chat-telegram-topic-state-is-durable", + "gitops.kustomizations-reconcile-cleanly", + "routing.provider-effort-and-fallback-evidence", + "scopes.agent-profile-is-distinct", + ], +) +def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None: + assert BY_ID[identifier].mandatory is True diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py new file mode 100644 index 00000000..09cc5d63 --- /dev/null +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -0,0 +1,271 @@ +"""Contracts for the separately armed, self-cleaning mutation mode.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +ephemeral = load_handoff_module("hermes_handoff_ephemeral") +exec_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") +model = load_handoff_module("hermes_handoff_model") + +REPO = "atlas/titan-iac" +TOKEN = "acceptance-20260817a" +VANTAGE = exec_module.operator_vantage() + + +def request(**overrides) -> object: + fields = { + "repo": REPO, + "remote": "origin", + "token": TOKEN, + "confirmation": ephemeral.CONFIRMATION, + "base": "main", + } + fields.update(overrides) + return ephemeral.ArmRequest(**fields) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def armed_runner(results: list) -> tuple: + spawn = FakeSpawn(results) + runner = exec_module.Runner( + mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + ) + return (runner, spawn) + + +def by_id(results: list) -> dict: + return {result.spec.id: result for result in results} + + +def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: + assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" + + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] +) +def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + + +def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: + ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") + + +@pytest.mark.parametrize( + ("overrides", "fragment"), + [ + ({"confirmation": "yes please"}, "confirmation phrase"), + ({"confirmation": ""}, "confirmation phrase"), + ({"repo": "atlas/other"}, "is not the expected"), + ({"token": "short"}, "ephemeral token"), + ({"token": "Has-Capitals-Here"}, "ephemeral token"), + ({"token": ""}, "ephemeral token"), + ({"base": ""}, "base branch is required"), + ], +) +def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: + with pytest.raises(ephemeral.ArmingError) as caught: + ephemeral.preflight(request(**overrides), REPO) + assert fragment in str(caught.value) + + +def test_preflight_accepts_a_correctly_armed_request() -> None: + ephemeral.preflight(request(), REPO) + + +@pytest.mark.parametrize( + ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] +) +def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: + assert ephemeral.normalise_branch(raw) == expected + + +def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: + """A guard checked once is a guard nobody notices regressing.""" + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + guard = results["ephemeral.protected-branch-refusal"] + + assert guard.status == model.PASS + assert guard.evidence["accepted"] == [] + assert "main" in guard.evidence["refused"] + + +def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: + """If the guard ever stops refusing, the armed run says so instead of pushing.""" + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.protected-branch-refusal" + ] + + assert guard.status == model.FAIL + assert "main" in guard.evidence["accepted"] + assert "the guard accepted" in guard.reason + + +def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: + runner, spawn = armed_runner( + [ + completed(stdout=""), # ls-remote: ref unused + completed(stdout=""), # push + completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft + completed(stdout=""), # close + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + completed(stdout=json.dumps({"number": 42, "state": "closed"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.PASS + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 + assert results["ephemeral.cleanup-verified"].status == model.PASS + assert all(result.spec.mandatory for result in results.values()) + assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) + + pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] + assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert "--force" not in pushed[0]["argv"] + + titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] + assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") + + +def test_an_existing_ref_stops_the_run_before_it_writes() -> None: + runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert "already exists" in results["ephemeral.feature-branch-push"].reason + assert "ephemeral.draft-pull-request" not in results + assert len(spawn.calls) == 1 + + +def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: + runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN + + +def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stderr="denied", returncode=1), + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_a_draft_response_without_a_number_is_a_failure() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout="{}"), + completed(stdout=""), + completed(stdout=""), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.draft-pull-request"].status == model.FAIL + + +def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stderr="cannot close", returncode=1), + completed(stderr="cannot delete", returncode=1), + completed(stdout="abc123\trefs/heads/x\n"), + completed(stdout=json.dumps({"number": 7, "state": "open"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + cleanup = results["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not be closed" in cleanup.reason + assert "still exists" in cleanup.reason + assert "still open" in cleanup.reason + + +def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stdout=""), + completed(stdout=""), + subprocess.TimeoutExpired(cmd="git", timeout=1.0), + completed(stderr="gone", returncode=1), + ] + ) + + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not verify removal" in cleanup.reason + assert "could not confirm pull request #7 is closed" in cleanup.reason + + +def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: + runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) + identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} + + assert identifiers <= { + "ephemeral.protected-branch-refusal", + "ephemeral.feature-branch-push", + "ephemeral.draft-pull-request", + "ephemeral.cleanup-verified", + } + + +def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: + """The mode, not the caller, is what opens the mutations.""" + spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) + runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + push = results["ephemeral.feature-branch-push"] + + assert push.status == model.FAIL + assert "policy" in push.reason + assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py new file mode 100644 index 00000000..2e340c7b --- /dev/null +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -0,0 +1,201 @@ +"""Contracts for the text-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +REVIEW = model.REVIEW +ATTEMPT = model.ATTEMPT +FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) + + +def deny_spec() -> object: + return spec( + "denied", + steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), + ) + + +def evaluate(built, outcomes): + return evaluators.evaluate(built, outcomes) + + +def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.PASS + assert evaluation.evidence == {"review": "no", "attempt": "refused"} + + +def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.FAIL + assert "reported yes" in evaluation.reason + + +def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + + assert evaluation.status == model.FAIL + assert "must be refused" in evaluation.reason + + +def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: + broken = outcome(stderr="connection refused", returncode=7) + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + + assert evaluation.status == model.NOT_RUN + assert "recognisable refusal" in evaluation.reason + + +def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: + """An authorization review states policy; only an attempt proves enforcement.""" + review_only = spec("denied", steps=(step("review", kind=REVIEW),)) + + evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) + + assert evaluation.status == model.NOT_RUN + assert "require a real attempt" in evaluation.reason + + +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: + built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) + + assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + + denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) + assert denied.status == model.FAIL and "reported no" in denied.reason + + failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + assert failed.status == model.FAIL and "step read failed" in failed.reason + + +def test_names_absent_flags_exact_names_and_substrings() -> None: + built = spec( + "names_absent", + {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, + steps=(step("env"),), + ) + + clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) + assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 + + leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + assert leaked.status == model.FAIL + assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] + + unreadable = evaluate(built, {"env": outcome(returncode=1)}) + assert unreadable.status == model.NOT_RUN + + +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: + built = spec( + "names_present", + {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + steps=(step("init"),), + ) + + good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") + assert evaluate(built, {"init": good}).status == model.PASS + + partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) + assert partial.status == model.FAIL and "@sha256:" in partial.reason + + empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) + assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN + + +def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: + single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS + + mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) + assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" + + several = spec( + "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + ) + assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS + + assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_distinct_count_separates_replica_count_from_actual_spread() -> None: + built = spec( + "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + ) + + spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) + assert spread.status == model.PASS and spread.evidence["distinct"] == 3 + + stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) + assert stacked.status == model.FAIL and "distinct values" in stacked.reason + + extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) + assert over.status == model.FAIL and "expected exactly 2" in over.reason + + assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN + assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: + built = spec( + "vantages_agree", + {"steps": ("operator", "self")}, + steps=(step("operator", vantage="operator"), step("self")), + ) + + agree = evaluate( + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + ) + assert agree.status == model.PASS + + differ = evaluate( + built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + ) + assert differ.status == model.FAIL and "disagree" in differ.reason + + empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + assert empty.status == model.NOT_RUN + + broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) + assert broken.status == model.NOT_RUN + + +def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: + assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + + +def test_lines_match_catches_a_blank_leading_column() -> None: + built = spec( + "lines_match", + {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + steps=(step("s"),), + ) + table = "Preview Src ID\n-------\nwork on task cli 2026\n" + + assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS + + blank = "Preview Src ID\n-------\n cli 2026\n" + failed = evaluate(built, {"s": outcome(stdout=blank)}) + assert failed.status == model.FAIL and failed.evidence["offenders"] + + assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: + built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) + + evaluation = evaluate(built, {}) + + assert evaluation.status == model.NOT_RUN + assert evaluation.evidence == {"armed": False} + assert evaluation.reason == "ephemeral mutation mode is not armed" + assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py new file mode 100644 index 00000000..cd8fc7d2 --- /dev/null +++ b/testing/tests/test_hermes_handoff_exec.py @@ -0,0 +1,173 @@ +"""Contracts for bounded, screened command execution in the handoff harness.""" + +from __future__ import annotations + +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +runner_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def build(results: list, **kwargs) -> tuple: + clock = FakeClock() + spawn = FakeSpawn(results) + runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + return (runner, spawn, clock) + + +OPERATOR = runner_module.operator_vantage() + + +def test_a_successful_command_is_captured_and_screened() -> None: + runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ok and outcome.ran + assert "[redacted]" in outcome.stdout + assert outcome.returncode == 0 + assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert runner.commands_run == 1 + + +def test_a_failing_command_is_not_ok_but_still_ran() -> None: + runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ran and not outcome.ok + assert "Forbidden" in outcome.combined + + +def test_a_policy_violation_never_reaches_a_subprocess() -> None: + runner, spawn, _ = build([]) + + outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR) + + assert outcome.error and outcome.error.startswith("policy:") + assert not outcome.ran + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_a_timeout_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error == runner_module.TIMEOUT_ERROR + assert not outcome.ran + + +def test_a_missing_binary_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error and outcome.error.startswith("spawn:") + + +def test_the_run_deadline_stops_further_probes_without_spawning() -> None: + runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) + + assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + clock.advance(11.0) + second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + + assert second.error == runner_module.DEADLINE_ERROR + assert len(spawn.calls) == 1 + assert runner.remaining_seconds < 0 + + +def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: + runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + clock.advance(6.0) + + runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert spawn.calls[0]["timeout"] == 4.0 + + +def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: + runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + + small = runner.run(("kubectl", "get", "pods"), OPERATOR) + large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + + assert small.truncated + assert not large.truncated + + +def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: + source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + environment = runner_module.build_environment(OPERATOR, source) + + assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert "ANTHROPIC_API_KEY" not in environment + + +def test_a_vantage_can_override_the_environment_it_needs() -> None: + vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml") + environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"}) + + assert environment["KUBECONFIG"] == "/tmp/operator.yaml" + assert "kubeconfig-pinned" in vantage.description + + +def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: + vantage = runner_module.operator_vantage(context="atlas-operator") + + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("git", "status")) == ("git", "status") + assert vantage.wrap(()) == () + + +def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None: + operator = runner_module.operator_vantage(context="atlas-operator") + vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) + + wrapped = vantage.wrap(("kubectl", "get", "pods")) + + assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") + assert wrapped[3] == "exec" + assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" + policy.check_argv(wrapped) + + +def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None: + vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes") + + assert vantage.wrap(("kubectl", "version"))[0] == "kubectl" + assert vantage.env == {} + + +def test_outcome_serialisation_screens_the_command_line() -> None: + outcome = runner_module.Outcome( + argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"), + vantage="operator", + returncode=0, + stdout="ok", + ) + + payload = outcome.as_dict() + + assert "a-long-enough-secret" not in payload["command"] + assert payload["vantage"] == "operator" + assert payload["stdout"] == "ok" + + +def test_combined_output_joins_only_the_streams_that_carry_text() -> None: + assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" + assert runner_module.Outcome(argv=(), vantage="x").combined == "" diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py new file mode 100644 index 00000000..9bb5c673 --- /dev/null +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -0,0 +1,279 @@ +"""Contracts for the JSON-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import datetime as dt +import json + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) + + +def evaluate(built, payload): + body = payload if isinstance(payload, str) else json.dumps(payload) + return evaluators.evaluate(built, {"s": outcome(stdout=body)}) + + +def one(rule: str, expect: dict): + return spec(rule, {"step": "s", **expect}, steps=(step("s"),)) + + +def test_json_field_compares_paths_against_expected_values() -> None: + built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + + assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + + wrong = evaluate(built, {"authenticated": False, "transport": "c"}) + assert wrong.status == model.FAIL + assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason + + assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN + + +def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: + built = one("json_field", {"fields": {"a": 1}}) + + assert evaluate(built, "not json").status == model.NOT_RUN + failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN + + +def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: + built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}}) + + assert evaluate(built, {"latency_ms": 500}).status == model.PASS + assert evaluate(built, {"latency_ms": 1}).status == model.PASS + + low = evaluate(built, {"latency_ms": 0}) + assert low.status == model.FAIL and "below minimum" in low.reason + + high = evaluate(built, {"latency_ms": 5000}) + assert high.status == model.FAIL and "above maximum" in high.reason + + assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN + assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_json_recent_treats_stale_evidence_as_a_failure() -> None: + built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) + + fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) + assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + + stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) + assert stale.status == model.FAIL and "limit 3600s" in stale.reason + + naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) + assert naive.status == model.PASS + + assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: + built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + payload = [ + {"number": 14, "merged": True}, + {"number": 15, "merged": False}, + {"number": 16, "merged": False}, + ] + + evaluation = evaluate(built, payload) + + assert evaluation.status == model.FAIL + assert "15: merged=False" in evaluation.reason + assert "16" not in evaluation.reason + assert evaluation.evidence["examined"] == [14, 15] + + +def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: + keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + absent = evaluate(keyed, [{"number": 99, "merged": True}]) + assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason + + unkeyed = one("all_of_field", {"fields": {"status": ["done"]}}) + assert evaluate(unkeyed, []).status == model.NOT_RUN + assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN + + +def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: + built = spec( + "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + ) + + evaluation = evaluate(built, {"other": []}) + + assert evaluation.status == model.NOT_RUN + assert "array path is absent" in evaluation.reason + + +def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None: + built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}}) + + evaluation = evaluate(built, [{"name": "a"}]) + + assert evaluation.status == model.FAIL + assert "suspended=None" in evaluation.reason + + +def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: + exempted = one( + "all_of_field", + {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + ) + payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + assert evaluate(exempted, payload).status == model.PASS + + non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) + assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS + blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}]) + assert blank.status == model.FAIL and "is empty" in blank.reason + + +def test_json_record_captures_named_paths_or_skips() -> None: + built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) + payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + + evaluation = evaluate(built, payload) + assert evaluation.status == model.PASS + assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]} + + missing = evaluate(built, [{"number": 14}]) + assert missing.status == model.NOT_RUN and "absent" in missing.reason + + +def routing_log(*records: dict) -> str: + return "\n".join(json.dumps(record) for record in records) + + +def routing_spec(**extra): + return spec( + "routing_evidence", + { + "step": "s", + "providers": ("codex", "claude"), + "efforts": ("high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + **extra, + }, + steps=(step("s"),), + ) + + +FULL_LOG = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, + {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, + {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, +) + + +def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None: + evaluation = evaluate(routing_spec(), FULL_LOG) + + assert evaluation.status == model.PASS + assert evaluation.evidence["providers"] == ["claude", "codex"] + assert evaluation.evidence["efforts"] == ["high", "xhigh"] + assert evaluation.evidence["lanes"] == ["route", "worker"] + assert evaluation.evidence["fallback_reasons"] == ["unavailable"] + assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3 + + +def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: + partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + + evaluation = evaluate(routing_spec(), partial) + + assert evaluation.status == model.FAIL + assert "provider:claude" in evaluation.reason + assert "effort:xhigh" in evaluation.reason + assert "lane:worker" in evaluation.reason + assert "fallback:none-observed" in evaluation.reason + + +def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: + clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG + + assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), "").status == model.NOT_RUN + assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN + + +def test_routing_evidence_can_require_freshness() -> None: + fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) + assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + + stale = evaluate( + routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG + ) + assert stale.status == model.FAIL and "freshness" in stale.reason + + undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) + assert unknown.status == model.NOT_RUN + + +def flux_spec(*expected: str): + return spec( + "flux_health", + {"step": "s", "expected_suspensions": expected}, + steps=(step("s"),), + ) + + +def test_flux_health_separates_parked_objects_from_broken_ones() -> None: + rows = ( + "flux-system/core\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\ttrue\tReady=False,\n" + "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + ) + + evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] + assert evaluation.evidence["unexpected_suspensions"] == [] + assert "unhealthy: flux-system/broken" in evaluation.reason + + +def test_flux_health_flags_a_suspension_nobody_declared() -> None: + rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] + + +def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: + """Flux drops Ready to False for the duration of a reconcile.""" + rows = ( + "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t\tReady=Unknown,\n" + ) + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.PASS + assert evaluation.evidence["total"] == 3 + + +def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: + padded = "\n\n" + FULL_LOG + "\n\n" + assert evaluate(routing_spec(), padded).status == model.PASS + + rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 + + +def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: + assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN + assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py new file mode 100644 index 00000000..37c3f9b9 --- /dev/null +++ b/testing/tests/test_hermes_handoff_model.py @@ -0,0 +1,180 @@ +"""Contracts for the handoff harness result model, verdict, and rendering.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +model = load_handoff_module("hermes_handoff_model") + + +def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): + return model.CheckResult( + spec=model.CheckSpec( + id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory + ), + status=status, + reason=f"{status} reason", + ) + + +def report(*results, errors: tuple[str, ...] = ()) -> model.Report: + return model.Report( + mode="read-only", + started_at="2026-08-17T00:00:00Z", + results=list(results), + harness_errors=list(errors), + ) + + +@pytest.mark.parametrize( + ("status", "mandatory", "blocking"), + [ + (model.PASS, True, False), + (model.NOT_APPLICABLE, True, False), + (model.FAIL, True, True), + (model.NOT_RUN, True, True), + (model.FAIL, False, False), + (model.NOT_RUN, False, False), + ], +) +def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None: + assert result(status, mandatory=mandatory).blocking is blocking + + +def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: + """The whole point of the four-valued classification.""" + assert report(result(model.NOT_RUN)).decision == model.NO_GO + assert report(result(model.FAIL)).decision == model.NO_GO + assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO + + +def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: + clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + assert clean.decision == model.NO_GO + + +def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: + """An empty catalog has nothing blocking; the harness errors are what stop it.""" + assert report().decision == model.GO + assert report(errors=("no checks were built",)).decision == model.NO_GO + + +def test_blocking_results_are_ordered_failures_before_skips() -> None: + ordered = report( + result(model.NOT_RUN, "b.skip"), + result(model.FAIL, "z.fail"), + result(model.PASS, "a.pass"), + ).blocking + + assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"] + + +def test_counts_and_group_summary_tally_every_status() -> None: + built = report( + result(model.PASS, "a", group="alpha"), + result(model.FAIL, "b", group="alpha"), + result(model.NOT_RUN, "c", group="beta"), + ) + + assert built.counts == { + model.PASS: 1, + model.FAIL: 1, + model.NOT_RUN: 1, + model.NOT_APPLICABLE: 0, + } + assert built.group_summary()["alpha"][model.PASS] == 1 + assert built.group_summary()["beta"][model.NOT_RUN] == 1 + + +def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: + built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True) + ] + + payload = built.as_dict() + + assert payload["decision"] == model.NO_GO + assert payload["blocking"] == ["x.fail"] + assert payload["harness"] == model.HARNESS + assert payload["schema_version"] == model.SCHEMA_VERSION + assert payload["vantages"][0]["identity"] == "u" + assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"} + + +def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None: + built = model.CheckResult( + spec=spec("allowed", identifier="x", steps=(step("a"),)), + status=model.PASS, + evidence={"token": "a-long-enough-secret", "count": 3}, + outcomes=[outcome(stdout="fine")], + ) + + with_commands = built.as_dict() + without = built.as_dict(include_outcomes=False) + + assert with_commands["evidence"]["token"] == "[redacted]" + assert with_commands["evidence"]["count"] == 3 + assert len(with_commands["commands"]) == 1 + assert "commands" not in without + assert with_commands["vantages"] == ["self"] + + +def test_rationale_is_carried_only_when_a_check_has_one() -> None: + without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict() + assert "rationale" not in without + + annotated = model.CheckSpec( + id="x", title="t", group="g", rule="allowed", rationale="because" + ) + with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict() + assert with_reason["rationale"] == "because" + + +def test_distinct_vantages_are_reported_in_first_use_order() -> None: + built = spec( + "allowed", + steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + ) + assert built.vantages == ("operator", "self") + + +def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: + built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord(name="self", description="d", available=False), + ] + + summary = built.render_summary() + + assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}" + assert "vantage operator: available identity=u" in summary + assert "vantage self: unavailable identity=unknown" in summary + assert "harness error: collision" in summary + assert "x.fail" in summary + assert "alpha" in summary + + +def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None: + fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc) + assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z" + assert model.utc_now().endswith("Z") + + +def test_unscreened_fields_walks_the_whole_payload() -> None: + offenders = model.unscreened_fields( + { + "clean": "hermes", + "nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"}, + "items": [1, "Authorization: Bearer abcdefghijklmnop"], + } + ) + + assert offenders == ["$.nested.leak", "$.items[1]"] + assert model.unscreened_fields({"clean": "hermes"}) == [] + assert model.unscreened_fields(None) == [] diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py new file mode 100644 index 00000000..13fdc8b4 --- /dev/null +++ b/testing/tests/test_hermes_handoff_policy.py @@ -0,0 +1,159 @@ +"""Contracts for the fail-closed command policy of the handoff harness.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +policy = load_handoff_module("hermes_handoff_policy") + + +def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY, allow_impersonation=inner) + + +@pytest.mark.parametrize( + "argv", + [ + ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + ("kubectl", "get", "secrets", "--output=name"), + ("kubectl", "logs", "deploy/x", "--tail", "1"), + ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("flux", "get", "kustomizations", "--all-namespaces"), + ("helm", "list", "--all-namespaces"), + ("git", "merge-base", "--is-ancestor", "a", "b"), + ("git", "config", "--get", "remote.origin.url"), + ("hermes", "kanban", "list", "--json"), + ("hermes", "sessions", "list", "--source", "telegram"), + ("hermes", "status"), + (policy.GITEA_CLIENT, "GET", "/api/v1/user"), + ], +) +def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: + check(*argv) + + +@pytest.mark.parametrize( + ("argv", "fragment"), + [ + ((), "empty command"), + (("vault", "read", "kv/x"), "forbidden binary"), + (("curl", "https://example.dev"), "forbidden binary"), + (("rm", "-rf", "/"), "allowlist"), + (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), + (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), + (("kubectl", "get", "secrets", "-A"), "-o name"), + (("kubectl", "describe", "secrets", "x"), "-o name"), + (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), + (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), + (("kubectl", "exec", "pod"), "-- command boundary"), + (("kubectl", "exec", "pod", "--"), "inner command"), + (("kubectl",), "outside the read-only boundary"), + (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), + (("flux",), "requires a subcommand"), + (("helm", "upgrade", "x"), "outside the read-only boundary"), + (("git", "commit", "-m", "x"), "not permitted"), + (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), + (("git", "config", "user.name", "x"), "may only read"), + (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), + (("hermes",), "requires a subcommand"), + ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), + ((policy.GITEA_CLIENT, "GET"), "method and a path"), + (("sh", "-c", "rm -rf /"), "frozen template"), + (("sh", "echo"), "frozen template"), + (("git", "log", "/etc/shadow"), "credential path"), + ], +) +def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: + with pytest.raises(policy.PolicyError) as caught: + check(*argv) + assert fragment in str(caught.value) + + +def test_unknown_mode_is_refused() -> None: + with pytest.raises(policy.PolicyError, match="unknown mode"): + policy.check_argv(("kubectl", "get", "pods"), "whatever") + + +def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: + check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") + check("kubectl", "create", "configmap", "probe", "--dry-run=server") + + +def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: + """`-n get` must not make a delete look like a read.""" + with pytest.raises(policy.PolicyError, match="requires a dry run"): + check("kubectl", "delete", "-n", "get", "configmap", "x") + + +def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check("kubectl", "get", "namespaces", "--as", "system:admin") + check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + + +def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: + inner = policy.shell("path_readable", path="/tmp") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") + with pytest.raises(policy.PolicyError, match="allowlist"): + check("kubectl", "exec", "pod", "--", "bash", "-c", "id") + + +def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: + """The path scan stops at `--`; the tail is a reviewed template, not free text.""" + inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + + +def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: + check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--all", "origin", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="not permitted"): + check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + + +def test_every_frozen_template_renders_and_is_accepted() -> None: + parameters = { + "account": "hermes-agent", + "askpass": "/opt/coordinator/gitea_askpass.sh", + "fields": "state,model", + "limit": "200", + "path": "/host-etc/passwd", + "url": "https://scm.example.dev/atlas/repo.git", + } + for name in policy.SHELL_TEMPLATES: + required = policy.template_parameters(name) + argv = policy.shell(name, **{key: parameters[key] for key in required}) + policy.check_argv(argv) + assert argv[0] == "sh" + + +def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None: + with pytest.raises(policy.PolicyError, match="unknown shell template"): + policy.render_shell("nope") + with pytest.raises(policy.PolicyError, match="expects parameters"): + policy.render_shell("path_mode") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="/tmp; rm -rf /") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="$(id)") + + +def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: + argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored") + assert list(policy.positionals(argv)) == ["get", "pods"] + assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"] + + +def test_a_subcommand_must_precede_its_resource_arguments() -> None: + with pytest.raises(policy.PolicyError, match="must precede"): + check("kubectl", "pods", "get") diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py new file mode 100644 index 00000000..a472d82b --- /dev/null +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -0,0 +1,162 @@ +"""Contracts for the handoff harness credential screen and output bounding.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +redaction = load_handoff_module("hermes_handoff_redaction") + + +@pytest.mark.parametrize( + "text", + [ + "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + "Authorization: Bearer abcdefghijklmnop", + "cookie: session=abcdefghijklmnop", + "password=hunter2000secret", + "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ", + "https://user:supersecretvalue@scm.example.dev/x", + "vault token hvs.CAESIJabcdefghijklmnop", + "aws AKIAIOSFODNN7EXAMPLE", + ], +) +def test_credential_shapes_are_screened(text: str) -> None: + assert redaction.contains_credential_shape(text) + assert redaction.REDACTED in redaction.redact(text) + + +@pytest.mark.parametrize( + "text", + [ + "sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7", + "ab346f55509d584e457fe26cf90be3078f7a375c", + '"automountServiceAccountToken": false', + '"key": "kubernetes.io/arch"', + "namespace: hermes", + "replicas: 3", + "checked_at: 2026-08-17T09:17:07.224400+00:00", + ], +) +def test_ordinary_evidence_survives_screening(text: str) -> None: + assert redaction.redact(text) == text + assert not redaction.contains_credential_shape(text) + + +def test_screened_json_is_still_parsable_json() -> None: + """A screen that corrupts structure would break every downstream evaluator.""" + payload = { + "spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"}, + "data": {"password": "a-long-enough-secret-value"}, + } + screened = redaction.redact(json.dumps(payload, indent=2)) + reparsed = json.loads(screened) + + assert reparsed["spec"]["automountServiceAccountToken"] is True + assert reparsed["spec"]["key"] == "kubernetes.io/arch" + assert reparsed["data"]["password"] == redaction.REDACTED + + +def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: + assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + long_mixed = "aB3" + "x9Y2z_" * 10 + assert redaction.redact(long_mixed) == redaction.REDACTED + assert redaction.redact("f" * 64) == "f" * 64 + + +@pytest.mark.parametrize( + ("key", "sensitive"), + [ + ("ANTHROPIC_API_KEY", True), + ("gitea_token", True), + ("clientSecret", True), + ("Set-Cookie", True), + ("automountServiceAccountToken", True), + ("key", False), + ("namespace", False), + ("", False), + ], +) +def test_key_sensitivity_classification(key: str, sensitive: bool) -> None: + assert redaction.is_sensitive_key(key) is sensitive + + +@pytest.mark.parametrize( + ("value", "possible"), + [ + ("true", False), + ("False", False), + ("3", False), + ("12.5", False), + ("short", False), + ("", False), + ("a-long-enough-secret", True), + ('"another-long-secret"', True), + ("1234567890123", False), + ("-12.5e9", False), + ], +) +def test_value_shape_gates_screening(value: str, possible: bool) -> None: + assert redaction.could_hold_secret(value) is possible + + +def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: + text, truncated = redaction.bound("x" * 100, max_bytes=10) + assert truncated + assert text.startswith("x" * 10) + assert redaction.TRUNCATION_NOTE in text + + text, truncated = redaction.bound("short", max_bytes=64) + assert (text, truncated) == ("short", False) + + +def test_bound_with_no_budget_keeps_nothing() -> None: + assert redaction.bound("content", max_bytes=0) == ("", True) + assert redaction.bound("", max_bytes=0) == ("", False) + + +def test_bound_never_splits_a_multibyte_character() -> None: + text, truncated = redaction.bound("é" * 10, max_bytes=5) + assert truncated + assert text.startswith("éé") + + +def test_safe_text_bounds_before_it_screens() -> None: + """Bounding first is what stops a huge dump from outrunning the screen.""" + payload = "password=" + "a" * 200 + text, truncated = redaction.safe_text(payload, max_bytes=20) + assert truncated + assert redaction.REDACTED in text + + +def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: + scrubbed = redaction.scrub( + { + "token": "a-long-enough-secret", + "automountServiceAccountToken": False, + "items": [{"password": "another-long-secret"}, 3, None], + "note": ("tuple", "values"), + } + ) + + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["automountServiceAccountToken"] is False + assert scrubbed["items"][0]["password"] == redaction.REDACTED + assert scrubbed["items"][1] == 3 + assert scrubbed["items"][2] is None + assert scrubbed["note"] == ["tuple", "values"] + + +def test_screening_is_idempotent() -> None: + once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") + assert redaction.redact(once) == once + assert redaction.redact("") == "" + + +def test_normalise_key_strips_separators() -> None: + assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2" diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py new file mode 100644 index 00000000..11f09840 --- /dev/null +++ b/testing/tests/test_hermes_handoff_rules.py @@ -0,0 +1,135 @@ +"""Contracts for the shared evaluator primitives and dispatch.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +rules = load_handoff_module("hermes_handoff_rules") +model = load_handoff_module("hermes_handoff_model") +load_handoff_module("hermes_handoff_evaluators") + + +PAYLOAD = { + "a": {"b": [{"c": 1}, {"c": 2}]}, + "items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}], + "flag": True, +} + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("", PAYLOAD), + ("flag", True), + ("a.b[0].c", 1), + ("a.b[].c", [1, 2]), + ("a.b[]", [{"c": 1}, {"c": 2}]), + ("items[].number", [14, 15]), + ], +) +def test_dotted_paths_index_and_map(path: str, expected) -> None: + assert rules.dotted(PAYLOAD, path) == expected + + +@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: + with pytest.raises(KeyError): + rules.dotted(PAYLOAD, path) + + +def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: + """Kanban bodies routinely carry raw newlines; that is not a broken read.""" + assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" + with pytest.raises(ValueError, match="empty output"): + rules.parse_json(outcome(stdout=" ")) + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout="not json")) + + +def test_lines_drops_blank_rows_and_trims() -> None: + assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"] + + +def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: + built = spec("allowed", steps=(step("a"), step("b"))) + + never_run = rules.missing_steps(built, {}) + assert never_run.status == model.NOT_RUN + assert "never executed" in never_run.reason + + errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + assert errored.status == model.NOT_RUN + assert "did not run" in errored.reason + + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None + + +def test_an_optional_step_may_be_absent() -> None: + built = spec("allowed", steps=(step("a"), step("b", optional=True))) + assert rules.missing_steps(built, {"a": outcome()}) is None + + +@pytest.mark.parametrize( + ("stderr", "returncode", "denied"), + [ + ("Error from server (Forbidden): pods is forbidden", 1, True), + ("User cannot list resource", 1, True), + ("Gitea API returned HTTP 403", 1, True), + ("connection refused", 1, False), + ("", 0, False), + ], +) +def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: + assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied + + +def test_step_keys_selects_by_kind() -> None: + built = spec( + "denied", + steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")), + ) + assert rules.step_keys(built, model.REVIEW) == ("r",) + assert rules.step_keys(built, model.ATTEMPT) == ("a",) + assert rules.step_keys(built, model.READ) == ("x",) + + +def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: + built = spec("json_field", {"step": "s"}, steps=(step("s"),)) + + payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')}) + assert (payload, key) == ({"a": 1}, "s") + + failed = rules.parsed_step(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN and "failed" in failed.reason + + unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) + assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + + +def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: + evaluation = rules.evaluate(spec("no-such-rule"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown evaluator" in evaluation.reason + + +def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: + """A harness bug must never be indistinguishable from a clean release.""" + + @rules.evaluator("explodes-for-test") + def _explode(_spec, _outcomes): + raise RuntimeError("boom") + + evaluation = rules.evaluate(spec("explodes-for-test"), {}) + + assert evaluation.status == model.NOT_RUN + assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason + del rules.EVALUATORS["explodes-for-test"] + + +def test_is_list_distinguishes_arrays_from_strings() -> None: + assert rules.is_list([1]) and rules.is_list(()) + assert not rules.is_list("text") + assert not rules.is_list(b"bytes") + assert not rules.is_list({"a": 1}) diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py new file mode 100644 index 00000000..379bb27d --- /dev/null +++ b/testing/tests/test_hermes_handoff_run.py @@ -0,0 +1,320 @@ +"""Contracts for handoff acceptance orchestration: vantages, catalog, report.""" + +from __future__ import annotations + +import datetime as dt +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, + step, +) + +harness_run = load_handoff_module("hermes_handoff_run") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def whoami(username: str) -> object: + return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + + +def runner_for(results: list, **kwargs): + return exec_module.Runner( + clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + ) + + +def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): + return model.CheckSpec( + id=identifier, + title="t", + group="g", + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + ) + + +def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: + specs = harness_run.build_catalog(TARGETS) + assert harness_run.validate_catalog(specs) == [] + groups = {spec.group for spec in specs} + assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups + assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + + +def test_catalog_validation_names_every_structural_problem() -> None: + broken = [ + check("no-such-rule", identifier="a.dup"), + check("allowed", identifier="a.dup"), + check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), + check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), + check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + model.CheckSpec( + id="e.impersonation", + title="t", + group="g", + rule="allowed", + steps=( + model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + ), + ), + ] + + problems = " | ".join(harness_run.validate_catalog(broken)) + + assert "duplicate check id" in problems + assert "unknown rule" in problems + assert "unknown vantage" in problems + assert "unknown kind" in problems + assert "needs a real attempt" in problems + assert "impersonates from the operator vantage" in problems + + +def test_identity_resolution_extracts_only_the_username() -> None: + runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) + username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") + + failed = harness_run.resolve_identity( + runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + ) + assert failed[0] == "" and "no access" in failed[1] + + unparsable = harness_run.resolve_identity( + runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + ) + assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + + nameless = harness_run.resolve_identity( + runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + ) + assert nameless[0] == "" and "no username" in nameless[1] + + +def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stderr="no pods", returncode=1), + completed(stdout="pod/hermes-chat-tenant-0\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" + assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick + assert by_name[catalog.NODE].available is False + assert by_name[catalog.CHAT].available is True + + +def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: + runner = runner_for([completed(stderr="connection refused", returncode=1)]) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + + assert set(vantages) == {catalog.OPERATOR} + assert records[0].available is False + assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + + +def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stdout="pod/hermes-node-1\n"), + completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + chat = next(record for record in records if record.name == catalog.CHAT) + + assert catalog.CHAT not in vantages + assert chat.available is False and "not found" in chat.detail + + +def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR} + assert "no running pod matches" in by_name[catalog.SELF].detail + + +def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: + """Corroboration needs two identities; one identity twice is a restatement.""" + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + ] + + problems = harness_run.vantage_problems(records) + + assert len(problems) == 1 + assert "same principal" in problems[0] + + +def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: + operator = model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ) + + absent = harness_run.vantage_problems([operator]) + assert "in-pod Hermes vantage is unavailable" in absent[0] + + anonymous = harness_run.vantage_problems( + [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + ) + assert "did not report an identity" in anonymous[0] + + clean = harness_run.vantage_problems( + [ + operator, + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + ] + ) + assert clean == [] + + +def test_a_check_whose_vantage_is_absent_is_not_run() -> None: + spec = check("allowed", steps=(step("s", vantage=catalog.NODE),)) + + result = harness_run.run_check(runner_for([]), spec, {}) + + assert result.status == model.NOT_RUN + assert "vantage node is unavailable" in result.reason + assert result.outcomes[0].error == "vantage node is unavailable" + + +def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: + spec = check( + "stdout_matches", + {"step": "s", "equals": "locked"}, + steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + ) + runner = runner_for([completed(stdout="locked", stderr="noise")]) + + result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + + assert result.status == model.PASS + assert result.outcomes[0].stdout == harness_run.UNRECORDED + assert result.outcomes[0].stderr == harness_run.UNRECORDED + + +def test_an_expired_deadline_leaves_the_report_saying_so() -> None: + clock = FakeClock() + runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + clock.advance(5.0) + spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) + + report = harness_run.build_report( + runner, + TARGETS, + [spec], + {catalog.OPERATOR: exec_module.operator_vantage()}, + [], + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert report.results[0].status == model.NOT_RUN + assert any("deadline expired" in error for error in report.harness_errors) + assert report.decision == model.NO_GO + + +def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: + runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) + specs = [ + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + ] + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + ] + + report = harness_run.build_report( + runner, + TARGETS, + specs, + {catalog.OPERATOR: exec_module.operator_vantage()}, + records, + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert [result.spec.id for result in report.results] == ["a.first", "z.last"] + assert report.baseline["baseline_commit"] == TARGETS.baseline_commit + assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.decision == model.GO + assert report.finished_at + + +def test_armed_results_replace_the_matching_catalog_placeholders() -> None: + placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + armed = model.CheckResult( + spec=model.CheckSpec( + id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + ), + status=model.PASS, + reason="pushed", + ) + + report = harness_run.build_report( + runner_for([]), + TARGETS, + [placeholder], + {}, + [], + "ephemeral-armed", + "2026-08-17T00:00:00Z", + [armed], + ) + + assert len(report.results) == 1 + assert report.results[0].status == model.PASS + + +def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None: + report = harness_run.unavailable_report( + "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS + ) + + assert report.decision == model.NO_GO + assert report.results[0].spec.id == "harness.startup" + assert report.results[0].status == model.NOT_RUN + assert report.harness_errors == ["arming refused: bad phrase"] diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py new file mode 100644 index 00000000..64964a27 --- /dev/null +++ b/testing/tests/test_hermes_handoff_support.py @@ -0,0 +1,110 @@ +"""Shared loader and fixtures for the handoff acceptance harness tests. + +The harness lives in ``scripts/ops`` as sibling modules rather than an installed +package, so tests import it the way the entry point does: by putting that +directory on the path once and importing by name. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPS = REPO_ROOT / "scripts" / "ops" + +if str(OPS) not in sys.path: + sys.path.insert(0, str(OPS)) + + +def load_handoff_module(name: str) -> Any: + """Import one harness module by name.""" + return importlib.import_module(name) + + +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + + +def outcome( + stdout: str = "", + stderr: str = "", + returncode: int | None = 0, + error: str | None = None, + vantage: str = "self", + argv: tuple[str, ...] = ("kubectl", "get", "pods"), +) -> Any: + """Return a synthetic command outcome for evaluator tests.""" + return exec_module.Outcome( + argv=argv, + vantage=vantage, + returncode=returncode, + stdout=stdout, + stderr=stderr, + error=error, + ) + + +def spec( + rule: str, + expect: dict | None = None, + steps: tuple = (), + identifier: str = "test.check", + mandatory: bool = True, + scope: str | None = None, +) -> Any: + """Return a synthetic check specification.""" + return model.CheckSpec( + id=identifier, + title="test check", + group="test", + rule=rule, + steps=steps, + expect=expect or {}, + mandatory=mandatory, + scope=scope or model.ALWAYS, + ) + + +def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: + """Return a synthetic step.""" + return model.Step( + key=key, + vantage=vantage, + argv=("kubectl", "get", "pods"), + kind=kind or model.READ, + optional=optional, + ) + + +class FakeClock: + """A monotonic clock the tests advance by hand.""" + + def __init__(self, start: float = 0.0) -> None: + self.now = start + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + """Move the clock forward.""" + self.now += seconds + + +class FakeSpawn: + """A ``subprocess.run`` stand-in driven by a queue of scripted results.""" + + def __init__(self, results: list) -> None: + self.results = list(results) + self.calls: list[dict] = [] + + def __call__(self, argv, **kwargs): + self.calls.append({"argv": list(argv), **kwargs}) + if not self.results: + raise AssertionError(f"unexpected command: {argv}") + outcome_or_error = self.results.pop(0) + if isinstance(outcome_or_error, Exception): + raise outcome_or_error + return outcome_or_error From f04a524738f5156df2b3dfb74ef37d8b6c71dd66 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 19:57:54 -0300 Subject: [PATCH 03/46] hermes: preserve terminal results and prevent duplicate fanout --- dockerfiles/Dockerfile.hermes-agent | 9 + .../Dockerfile.hermes-agent.dockerignore | 2 + .../hermes-execution-safety-regression.py | 212 +++++++++++++ dockerfiles/patch-hermes-execution-safety.py | 150 +++++++++ services/hermes/scripts/cli_lane_runner.py | 294 +++++++++++++++++- testing/tests/test_hermes_cli_lanes.py | 294 +++++++++++++++++- 6 files changed, 942 insertions(+), 19 deletions(-) create mode 100644 dockerfiles/hermes-execution-safety-regression.py create mode 100644 dockerfiles/patch-hermes-execution-safety.py diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index ce9ceb46..fc99307e 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -1544,6 +1544,13 @@ function RootRedirect() { } NODE +COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py +COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py +RUN /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \ + && /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \ + && rm /tmp/patch-hermes-execution-safety.py \ + /tmp/hermes-execution-safety-regression.py + COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate RUN cd /opt/hermes/web \ @@ -1602,6 +1609,8 @@ RUN cd /opt/hermes/web \ /opt/hermes/agent/turn_context.py \ /opt/hermes/agent/conversation_loop.py \ /opt/hermes/hermes_cli/oneshot.py \ + /opt/hermes/hermes_cli/kanban_decompose.py \ + /opt/hermes/gateway/kanban_watchers.py \ /opt/hermes/tools/delegate_tool.py \ /opt/hermes/tools/web_tools.py \ /opt/hermes/tools/python_sandbox_tool.py \ diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index 1327bda4..1b3f891a 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -5,3 +5,5 @@ !dockerfiles/hermes-public-extract/** !dockerfiles/hermes-session-activity-panel.tsx !dockerfiles/hermes-session-migrate.py +!dockerfiles/patch-hermes-execution-safety.py +!dockerfiles/hermes-execution-safety-regression.py diff --git a/dockerfiles/hermes-execution-safety-regression.py b/dockerfiles/hermes-execution-safety-regression.py new file mode 100644 index 00000000..e21bd703 --- /dev/null +++ b/dockerfiles/hermes-execution-safety-regression.py @@ -0,0 +1,212 @@ +"""Build-time regressions for Hermes automatic decomposition safety.""" + +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from types import SimpleNamespace +from unittest import mock + + +_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-") +os.environ["HERMES_HOME"] = _HERMES_HOME.name + +from agent import auxiliary_client # noqa: E402 +from hermes_cli import kanban_db # noqa: E402 +from hermes_cli import kanban_decompose # noqa: E402 + + +class _FakeCompletions: + def __init__(self, payload: dict, before_response=None) -> None: + self.payload = payload + self.before_response = before_response + self.calls = 0 + + def create(self, **_kwargs): + self.calls += 1 + if self.before_response is not None: + self.before_response() + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content=json.dumps(self.payload)) + ) + ] + ) + + +class _FakeClient: + def __init__(self, completions: _FakeCompletions) -> None: + self.chat = SimpleNamespace(completions=completions) + + +class AutomaticDecompositionSafetyTests(unittest.TestCase): + """Exercise the patched upstream APIs against an isolated real database.""" + + def setUp(self) -> None: + self.connection = kanban_db.connect() + self.patches = [ + mock.patch.object( + kanban_decompose, + "_build_roster", + return_value=([], {"default"}), + ), + mock.patch.object( + kanban_decompose, + "_load_config", + return_value={"kanban": {"auto_promote_children": True}}, + ), + mock.patch.object( + auxiliary_client, + "get_auxiliary_extra_body", + return_value={}, + ), + ] + for patch in self.patches: + patch.start() + + def tearDown(self) -> None: + for patch in reversed(self.patches): + patch.stop() + self.connection.close() + + def _client(self, payload: dict, before_response=None) -> _FakeCompletions: + completions = _FakeCompletions(payload, before_response) + client = _FakeClient(completions) + patch = mock.patch.object( + auxiliary_client, + "get_text_auxiliary_client", + return_value=(client, "test-decomposer"), + ) + patch.start() + self.addCleanup(patch.stop) + return completions + + def _status(self, task_id: str) -> str: + task = kanban_db.get_task(self.connection, task_id) + self.assertIsNotNone(task) + return task.status + + def _route_executed_task_to_triage(self) -> str: + task_id = kanban_db.create_task( + self.connection, + title="already implemented parent", + ) + for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT): + if attempt: + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="same unavailable capability", + kind="capability", + ) + ) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual( + len(kanban_db.list_runs(self.connection, task_id)), + kanban_db.BLOCK_RECURRENCE_LIMIT, + ) + return task_id + + def test_automatic_decomposer_skips_executed_triage_task(self) -> None: + task_id = self._route_executed_task_to_triage() + completions = self._client( + {"fanout": False, "title": "must not run", "body": "must not run"} + ) + before = len(kanban_db.list_tasks(self.connection)) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("execution history", outcome.reason) + self.assertEqual(completions.calls, 0) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) + + def test_manual_decomposition_remains_available_after_execution(self) -> None: + task_id = self._route_executed_task_to_triage() + completions = self._client( + { + "fanout": False, + "title": "operator-approved retry", + "body": "Retain one bounded task.", + "assignee": "default", + } + ) + + outcome = kanban_decompose.decompose_task(task_id, author="operator") + + self.assertTrue(outcome.ok) + self.assertFalse(outcome.fanout) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "ready") + + def test_fresh_triage_task_still_auto_promotes(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="fresh objective", + triage=True, + ) + completions = self._client( + { + "fanout": False, + "title": "specified objective", + "body": "One bounded task.", + "assignee": "default", + } + ) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertTrue(outcome.ok) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "ready") + + def test_execution_history_gained_during_llm_call_blocks_commit(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="concurrent triage objective", + triage=True, + ) + + def add_run() -> None: + with kanban_db.connect_closing() as connection: + kanban_db._synthesize_ended_run( + connection, + task_id, + outcome="reclaimed", + summary="concurrent execution evidence", + ) + + completions = self._client( + { + "fanout": True, + "tasks": [ + { + "title": "redundant child", + "body": "must not be created", + "assignee": "default", + "parents": [], + } + ], + }, + before_response=add_run, + ) + before = len(kanban_db.list_tasks(self.connection)) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("gained execution history", outcome.reason) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/dockerfiles/patch-hermes-execution-safety.py b/dockerfiles/patch-hermes-execution-safety.py new file mode 100644 index 00000000..bcf93c50 --- /dev/null +++ b/dockerfiles/patch-hermes-execution-safety.py @@ -0,0 +1,150 @@ +"""Patch upstream Hermes auto-decomposition to respect execution history.""" + +import os +from pathlib import Path + + +def replace_once(source: str, before: str, after: str, label: str) -> str: + """Replace one anchored upstream fragment and fail closed on source drift.""" + count = source.count(before) + if count != 1: + raise SystemExit( + f"Hermes {label} patch context changed: expected 1, found {count}" + ) + return source.replace(before, after, 1) + + +source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) +decompose_path = source_root / "hermes_cli/kanban_decompose.py" +decompose = decompose_path.read_text(encoding="utf-8") + +helper_anchor = ''' if chosen not in valid_names: + return default_assignee + return chosen + + +''' +helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool: + """Return whether a task has ever entered a worker run.""" + with kb.connect_closing() as conn: + return bool(kb.list_runs(conn, task_id)) + + +''' +decompose = replace_once( + decompose, + helper_anchor, + helper_replacement, + "auto-decompose execution-history helper", +) + +signature_before = '''def decompose_task( + task_id: str, + *, + author: Optional[str] = None, + timeout: Optional[int] = None, +) -> DecomposeOutcome: +''' +signature_after = '''def decompose_task( + task_id: str, + *, + author: Optional[str] = None, + timeout: Optional[int] = None, + automatic: bool = False, +) -> DecomposeOutcome: +''' +decompose = replace_once( + decompose, + signature_before, + signature_after, + "auto-decompose function signature", +) + +status_before = ''' if task.status != "triage": + return DecomposeOutcome( + task_id, False, f"task is not in triage (status={task.status!r})" + ) + + cfg = _load_config() +''' +status_after = ''' if task.status != "triage": + return DecomposeOutcome( + task_id, False, f"task is not in triage (status={task.status!r})" + ) + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task has execution history and requires deliberate manual triage", + ) + + cfg = _load_config() +''' +decompose = replace_once( + decompose, + status_before, + status_after, + "auto-decompose preflight guard", +) + +single_before = ''' with kb.connect_closing() as conn: + ok = kb.specify_triage_task( +''' +single_after = ''' if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) + with kb.connect_closing() as conn: + ok = kb.specify_triage_task( +''' +decompose = replace_once( + decompose, + single_before, + single_after, + "auto-decompose single-task commit guard", +) + +fanout_before = ''' try: + with kb.connect_closing() as conn: + child_ids = kb.decompose_triage_task( +''' +fanout_after = ''' if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) + try: + with kb.connect_closing() as conn: + child_ids = kb.decompose_triage_task( +''' +decompose = replace_once( + decompose, + fanout_before, + fanout_after, + "auto-decompose fanout commit guard", +) +decompose_path.write_text(decompose, encoding="utf-8") + + +watcher_path = source_root / "gateway/kanban_watchers.py" +watcher = watcher_path.read_text(encoding="utf-8") +call_before = ''' outcome = _decomp.decompose_task( + tid, author="auto-decomposer", + ) +''' +call_after = ''' outcome = _decomp.decompose_task( + tid, + author="auto-decomposer", + automatic=True, + ) +''' +watcher = replace_once( + watcher, + call_before, + call_after, + "gateway automatic-decomposition call", +) +watcher_path.write_text(watcher, encoding="utf-8") diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 3eba3453..c26f26ca 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -123,6 +123,10 @@ class ProcessResult: capacity_failure: bool +class TerminalFinalizationPending(RuntimeError): + """An accepted worker result is durable but not committed to Kanban yet.""" + + def utc_now() -> str: return datetime.now(timezone.utc).isoformat() @@ -293,6 +297,115 @@ def state_path(board: str, task_id: str) -> Path: return STATE_ROOT / safe_board / f"{safe_task}.json" +def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path: + """Return a unique provider-result path; completed turns are never reused.""" + safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) + return state_file.with_name( + f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json" + ) + + +def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path: + """Return the durable path for one exact structured worker response.""" + safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) + return state_file.with_name( + f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json" + ) + + +def _terminal_path(state_file: Path, run_id: Any) -> Path: + """Return the replay journal path for an accepted terminal response.""" + safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) + return state_file.with_name(f"{state_file.stem}.run-{safe_run}.terminal.json") + + +def _persist_candidate( + state: dict[str, Any], + state_file: Path, + structured: dict[str, Any], + *, + route: Route, + returncode: int, + goal_turn: int, +) -> Path: + """Persist every structured response before any judge can supersede it.""" + sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1 + path = _candidate_path(state_file, state.get("run_id"), sequence) + while path.exists(): + sequence += 1 + path = _candidate_path(state_file, state.get("run_id"), sequence) + atomic_json( + path, + { + "board": state.get("board"), + "task_id": state.get("task_id"), + "expected_run_id": state.get("run_id"), + "goal_turn": goal_turn, + "provider": route.provider, + "model": route.model, + "effort": route.effort, + "returncode": returncode, + "structured": structured, + "recorded_at": utc_now(), + }, + ) + state["candidate_sequence"] = sequence + state["last_candidate_file"] = str(path) + atomic_json(state_file, state) + return path + + +def _write_terminal_record( + state_file: Path, + *, + board: str, + task_id: str, + run_id: Any, + structured: dict[str, Any], + summary: str, + metadata: dict[str, Any], +) -> tuple[Path, dict[str, Any]]: + """Journal an accepted result before attempting the Kanban transaction.""" + path = _terminal_path(state_file, run_id) + record = { + "board": board, + "task_id": task_id, + "expected_run_id": run_id, + "result": json.dumps(structured, sort_keys=True), + "summary": summary, + "metadata": metadata, + "kanban_state": "pending", + "recorded_at": utc_now(), + } + atomic_json(path, record) + return path, record + + +def _terminal_record_valid(record: dict[str, Any]) -> bool: + """Reject malformed or non-terminal replay journals without side effects.""" + if not isinstance(record, dict): + return False + required_strings = ("board", "task_id", "result", "summary") + if not all( + isinstance(record.get(key), str) and record[key] + for key in required_strings + ): + return False + if not isinstance(record.get("expected_run_id"), int): + return False + if not isinstance(record.get("metadata"), dict): + return False + try: + structured = json.loads(record["result"]) + except (TypeError, json.JSONDecodeError): + return False + return ( + isinstance(structured, dict) + and structured.get("status") == "completed" + and not structured.get("blockers") + ) + + def build_prompt(context: str, workspace: Path, handoff: str = "") -> str: """Create a bounded worker contract with an explicit machine-readable result.""" return f"""You are a durable coding worker managed by Hermes Kanban. @@ -667,11 +780,22 @@ def run_provider( state["claude_session_id"] = str(uuid.uuid4()) state["current_route"] = asdict(route) state["updated_at"] = utc_now() - atomic_json(state_file, state) env = _base_env() - result_file = state_file.with_suffix(".result.json") if route.provider == "codex": - result_file.unlink(missing_ok=True) + result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1 + result_file = _result_path(state_file, state.get("run_id"), result_sequence) + while result_file.exists(): + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + result_file.parent.mkdir(parents=True, exist_ok=True) + result_file.touch(mode=0o600, exist_ok=False) + state["result_sequence"] = result_sequence + state["current_result_file"] = str(result_file) + atomic_json(state_file, state) command = _codex_command(route, prompt, workspace, state, result_file) result = stream_process( command, @@ -690,8 +814,23 @@ def run_provider( and NO_CODEX_THREAD in result.output.lower() ): state.pop("codex_thread_id", None) + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + while result_file.exists(): + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + result_file.touch(mode=0o600, exist_ok=False) + state["result_sequence"] = result_sequence + state["current_result_file"] = str(result_file) atomic_json(state_file, state) - result_file.unlink(missing_ok=True) result = stream_process( _codex_command(route, prompt, workspace, state, result_file), provider="codex", @@ -706,8 +845,13 @@ def run_provider( file_result = load_json(result_file) if file_result.get("status") in cli_lane_goal.RESULT_STATUSES: result.structured = file_result + try: + result_file.chmod(0o600) + except OSError: + pass return result + atomic_json(state_file, state) resume = bool(state.get("claude_started")) result = stream_process( _claude_command(route, prompt, state, resume), @@ -801,6 +945,95 @@ def _board_call( raise last_storage_error +def _finalize_terminal_record( + kanban_db: Any, + path: Path, + record: dict[str, Any] | None = None, +) -> str: + """Commit one exact-run terminal journal, or leave it safely pending.""" + document = dict(record or load_json(path)) + if not _terminal_record_valid(document): + return "invalid" + board = str(document["board"]) + task_id = str(document["task_id"]) + expected_run_id = int(document["expected_run_id"]) + + def operation(conn: Any) -> str: + task = kanban_db.get_task(conn, task_id) + if task is None: + return "stale" + status = str(_task_value(task, "status", "")) + if status == "done": + return ( + "committed" + if str(_task_value(task, "result", "") or "") == document["result"] + else "stale" + ) + if ( + status != "running" + or _task_value(task, "current_run_id", None) != expected_run_id + ): + return "stale" + completed = kanban_db.complete_task( + conn, + task_id, + result=document["result"], + summary=document["summary"], + metadata=document["metadata"], + expected_run_id=expected_run_id, + ) + return "committed" if completed else "pending" + + outcome = str(_board_call(kanban_db, board, operation)) + if outcome == "committed": + document["kanban_state"] = "committed" + document["committed_at"] = utc_now() + atomic_json(path, document) + return outcome + + +def recover_pending_finalizations() -> int: + """Replay accepted exact-run results before scheduling more provider work.""" + from hermes_cli import kanban_db + + recovered = 0 + for path in sorted(STATE_ROOT.glob("*/*.terminal.json")): + record = load_json(path) + if record.get("kanban_state") == "committed": + continue + try: + outcome = _finalize_terminal_record(kanban_db, path, record) + except Exception as error: + board = str(record.get("board") or "unknown") + _record_board_access_error(board, error) + continue + if outcome == "committed": + recovered += 1 + return recovered + + +def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool: + """Keep an exact run claimed while its accepted result awaits replay.""" + if not isinstance(run_id, int): + return False + path = _terminal_path(state_path(board, task_id), run_id) + try: + path.stat() + except FileNotFoundError: + return False + except OSError: + return True + record = load_json(path) + if not _terminal_record_valid(record): + return True + identity = ( + record.get("board") == board + and record.get("task_id") == task_id + and record.get("expected_run_id") == run_id + ) + return not identity or record.get("kanban_state") != "committed" + + def execute_claim(board: str, task_id: str) -> None: """Execute one already-claimed task and commit its outcome to Kanban.""" from hermes_cli import kanban_db @@ -980,6 +1213,16 @@ def execute_claim(board: str, task_id: str) -> None: workspace, structured.get("artifacts"), ) + candidate_file = _persist_candidate( + state, + state_file, + structured, + route=route, + returncode=result.returncode, + goal_turn=goal_turn, + ) + else: + candidate_file = None metadata = { "executor": "direct-cli-lane", "provider": route.provider, @@ -992,6 +1235,8 @@ def execute_claim(board: str, task_id: str) -> None: "goal_mode": goal_mode, "goal_turn": goal_turn, } + if candidate_file is not None: + metadata["candidate_file"] = str(candidate_file) if structured: for key in ( "changed_files", @@ -1036,18 +1281,27 @@ def execute_claim(board: str, task_id: str) -> None: and result.returncode == 0 and completion_problem is None ): - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.complete_task( - fresh, - task_id, - result=json.dumps(structured, sort_keys=True), - summary=str(structured.get("summary") or "Completed"), - metadata=metadata, - expected_run_id=run_id, - ), + terminal_file = _terminal_path(state_file, run_id) + metadata["terminal_record"] = str(terminal_file) + terminal_file, terminal_record = _write_terminal_record( + state_file, + board=board, + task_id=task_id, + run_id=run_id, + structured=structured, + summary=str(structured.get("summary") or "Completed"), + metadata=metadata, ) + outcome = _finalize_terminal_record( + kanban_db, + terminal_file, + terminal_record, + ) + if outcome != "committed": + raise TerminalFinalizationPending( + "accepted worker result is durably journaled but " + f"Kanban finalization is {outcome}" + ) break can_continue = ( @@ -1126,6 +1380,8 @@ def execute_claim(board: str, task_id: str) -> None: ), ) break + except TerminalFinalizationPending as error: + comment(str(error)) except Exception as error: failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}" _board_call( @@ -1178,6 +1434,7 @@ def recover_orphans() -> None: """Return external running tasks to ready after a runner/pod restart.""" from hermes_cli import kanban_db + recover_pending_finalizations() try: boards = kanban_db.list_boards(include_archived=False) except Exception as error: @@ -1194,9 +1451,13 @@ def recover_orphans() -> None: try: for task in kanban_db.list_tasks(conn): if _external(task) and str(_task_value(task, "status", "")) == "running": + task_id = str(_task_value(task, "id")) + run_id = _task_value(task, "current_run_id", None) + if _has_pending_finalization(board, task_id, run_id): + continue kanban_db.reclaim_task( conn, - str(_task_value(task, "id")), + task_id, reason="direct CLI lane restarted; provider session will resume", ) BOARD_CORRUPTION_ERRORS.pop(board, None) @@ -1275,6 +1536,7 @@ def main() -> int: except Exception as error: print(f"worker future failed: {error}", file=sys.stderr, flush=True) del futures[future] + recover_pending_finalizations() active = set(futures.values()) try: newly_claimed = claim_ready(active, workers - len(futures)) diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..b3f53f65 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -359,6 +359,66 @@ def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch) assert "codex_thread_id" not in json.loads(state_file.read_text()) +def test_codex_result_files_are_unique_and_prior_turns_are_retained( + tmp_path: Path, + monkeypatch, +): + state = {"run_id": 17} + state_file = tmp_path / "task.json" + result_paths = [] + + def fake_stream(command, **_kwargs): + result_path = Path(command[command.index("-o") + 1]) + result_path.write_text( + json.dumps( + { + "status": "completed", + "summary": f"turn {len(result_paths) + 1}", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + } + ), + encoding="utf-8", + ) + result_paths.append(result_path) + return lanes.ProcessResult(0, "", None, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ) + + first = lanes.run_provider( + route, + "First turn.", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + second = lanes.run_provider( + route, + "Second turn.", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + + assert first.structured["summary"] == "turn 1" + assert second.structured["summary"] == "turn 2" + assert len(set(result_paths)) == 2 + assert all(path.exists() for path in result_paths) + assert all(path.stat().st_mode & 0o777 == 0o600 for path in result_paths) + + def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path): result = lanes.stream_process( [sys.executable, "-c", "print('authentication work completed')"], @@ -594,6 +654,197 @@ def test_board_call_retries_storage_faults_on_fresh_connections(): assert all(connection.closed for connection in connections) +def test_accepted_result_survives_failed_finalization_and_replays_exact_run( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_terminal", + status="running", + result=None, + current_run_id=41, + assignee="cli-auto", + max_runtime_seconds=60, + ) + completions = [] + comments = [] + permit_completion = {"value": False} + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + completions.append(kwargs) + if not permit_completion["value"]: + return False + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish and verify the objective.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **_kwargs: pytest.fail( + "an accepted journaled result must not be converted into a block" + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + structured = { + "status": "completed", + "summary": "Verified exact terminal result.", + "changed_files": [], + "tests_run": ["pytest: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + } + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False), + ) + + lanes.execute_claim("cassandra", "t_terminal") + + candidates = list((state_root / "cassandra").glob("*.candidate-*.json")) + terminals = list((state_root / "cassandra").glob("*.terminal.json")) + assert len(candidates) == 1 + assert json.loads(candidates[0].read_text())["structured"] == structured + assert len(terminals) == 1 + assert json.loads(terminals[0].read_text())["kanban_state"] == "pending" + assert task.status == "running" + assert any("durably journaled" in body for body in comments) + + permit_completion["value"] = True + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert len(completions) == 2 + terminal = json.loads(terminals[0].read_text()) + assert terminal["kanban_state"] == "committed" + assert completions[-1]["result"] == terminal["result"] + + +def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + state_file = lanes.state_path("cassandra", "t_terminal") + terminal_path, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_terminal", + run_id=8, + structured={ + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + summary="done", + metadata={}, + ) + assert terminal_path.exists() + task = SimpleNamespace( + id="t_terminal", + status="running", + assignee="cli-auto", + current_run_id=8, + ) + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + list_tasks=lambda _conn: [task], + reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + + lanes.recover_orphans() + + assert reclaimed == [] + + +def test_terminal_replay_never_crosses_into_a_replacement_run( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + state_file = lanes.state_path("cassandra", "t_terminal") + terminal_path, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_terminal", + run_id=8, + structured={ + "status": "completed", + "summary": "old run", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + summary="old run", + metadata={}, + ) + replacement = SimpleNamespace( + id="t_terminal", + status="running", + result=None, + current_run_id=9, + ) + completions = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: replacement, + complete_task=lambda *_args, **_kwargs: completions.append(True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + assert completions == [] + assert json.loads(terminal_path.read_text())["kanban_state"] == "pending" + + @pytest.mark.parametrize( ("result", "expected_action"), [ @@ -640,6 +891,8 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts( ): task = SimpleNamespace( id="t_worker", + status="running", + result=None, current_run_id=4, assignee="cli-auto", max_runtime_seconds=60, @@ -677,7 +930,9 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts( heartbeats.append((note, expected_run_id)) or True ), add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) @@ -717,6 +972,8 @@ def test_goal_card_continues_after_local_judge_rejects_progress( ): task = SimpleNamespace( id="t_goal", + status="running", + result=None, current_run_id=12, assignee="cli-auto", max_runtime_seconds=300, @@ -741,7 +998,9 @@ def test_goal_card_continues_after_local_judge_rejects_progress( build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", heartbeat_worker=lambda *_args, **_kwargs: True, add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) @@ -828,6 +1087,14 @@ def test_goal_card_continues_after_local_judge_rejects_progress( assert route_calls[2][1]["exclude_provider"] == "claude" assert "prior rejected reports" in judge_contexts[1] assert "commit, push, and remote verification are missing" in judge_contexts[1] + candidates = sorted(tmp_path.glob("state.run-12.candidate-*.json")) + assert len(candidates) == 2 + assert json.loads(candidates[0].read_text())["structured"]["summary"] == ( + "Focused tests passed." + ) + assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith( + "Full tests passed" + ) assert reports == [] @@ -902,7 +1169,7 @@ def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: P set_workspace_path=lambda *_args: None, build_worker_context=lambda *_args: "resume objective", add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: None, + complete_task=lambda *_args, **_kwargs: True, block_task=lambda *_args, **_kwargs: None, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) @@ -1256,6 +1523,27 @@ def test_agent_dashboard_reconnects_all_transient_websockets(): assert "dashboard token rotated by a server restart" in dockerfile +def test_agent_image_runs_execution_safety_patch_and_regressions(): + dockerfiles = HERMES.parents[1] / "dockerfiles" + dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8") + dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text( + encoding="utf-8" + ) + + for name in ( + "patch-hermes-execution-safety.py", + "hermes-execution-safety-regression.py", + ): + assert f"COPY dockerfiles/{name}" in dockerfile + assert f"!dockerfiles/{name}" in dockerignore + assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in ( + dockerfile + ) + assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in ( + dockerfile + ) + + def test_agent_refreshes_routes_after_restoring_cli_logins(): deployment = _agent_deployment() init_containers = { From 64ea6d685c1cb7b84d4cc5a23cf3e29c391a0e8d Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:27:12 -0300 Subject: [PATCH 04/46] hermes: harden replay and decomposition races --- .../hermes-execution-safety-regression.py | 45 +- dockerfiles/patch-hermes-execution-safety.py | 167 ++++++- services/hermes/scripts/cli_lane_runner.py | 472 +++++++++++++++--- testing/tests/test_hermes_cli_lanes.py | 438 +++++++++++++++- 4 files changed, 1019 insertions(+), 103 deletions(-) diff --git a/dockerfiles/hermes-execution-safety-regression.py b/dockerfiles/hermes-execution-safety-regression.py index e21bd703..a632e51b 100644 --- a/dockerfiles/hermes-execution-safety-regression.py +++ b/dockerfiles/hermes-execution-safety-regression.py @@ -167,22 +167,45 @@ class AutomaticDecompositionSafetyTests(unittest.TestCase): self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "ready") - def test_execution_history_gained_during_llm_call_blocks_commit(self) -> None: + def _insert_run(self, task_id: str) -> None: + with kanban_db.connect_closing() as connection: + kanban_db._synthesize_ended_run( + connection, + task_id, + outcome="reclaimed", + summary="concurrent execution evidence", + ) + + def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="concurrent single objective", + triage=True, + ) + completions = self._client( + { + "fanout": False, + "title": "redundant rewrite", + "body": "must not be committed", + "assignee": "default", + }, + before_response=lambda: self._insert_run(task_id), + ) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("gained execution history", outcome.reason) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "triage") + + def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None: task_id = kanban_db.create_task( self.connection, title="concurrent triage objective", triage=True, ) - def add_run() -> None: - with kanban_db.connect_closing() as connection: - kanban_db._synthesize_ended_run( - connection, - task_id, - outcome="reclaimed", - summary="concurrent execution evidence", - ) - completions = self._client( { "fanout": True, @@ -195,7 +218,7 @@ class AutomaticDecompositionSafetyTests(unittest.TestCase): } ], }, - before_response=add_run, + before_response=lambda: self._insert_run(task_id), ) before = len(kanban_db.list_tasks(self.connection)) diff --git a/dockerfiles/patch-hermes-execution-safety.py b/dockerfiles/patch-hermes-execution-safety.py index bcf93c50..621d508d 100644 --- a/dockerfiles/patch-hermes-execution-safety.py +++ b/dockerfiles/patch-hermes-execution-safety.py @@ -15,6 +15,106 @@ def replace_once(source: str, before: str, after: str, label: str) -> str: source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) +db_path = source_root / "hermes_cli/kanban_db.py" +db = db_path.read_text(encoding="utf-8") + +specify_signature_before = '''def specify_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + title: Optional[str] = None, + body: Optional[str] = None, + assignee: Optional[str] = None, + author: Optional[str] = None, +) -> bool: +''' +specify_signature_after = '''def specify_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + title: Optional[str] = None, + body: Optional[str] = None, + assignee: Optional[str] = None, + author: Optional[str] = None, + require_no_runs: bool = False, +) -> bool: +''' +db = replace_once( + db, + specify_signature_before, + specify_signature_after, + "transactional specify signature", +) + +specify_guard_before = ''' if existing is None: + return False + sets: list[str] = ["status = 'todo'"] +''' +specify_guard_after = ''' if existing is None: + return False + if require_no_runs and conn.execute( + "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", + (task_id,), + ).fetchone() is not None: + return False + sets: list[str] = ["status = 'todo'"] +''' +db = replace_once( + db, + specify_guard_before, + specify_guard_after, + "transactional specify execution-history guard", +) + +decompose_signature_before = '''def decompose_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + root_assignee: Optional[str], + children: list[dict], + author: Optional[str] = None, + auto_promote: bool = True, +) -> Optional[list[str]]: +''' +decompose_signature_after = '''def decompose_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + root_assignee: Optional[str], + children: list[dict], + author: Optional[str] = None, + auto_promote: bool = True, + require_no_runs: bool = False, +) -> Optional[list[str]]: +''' +db = replace_once( + db, + decompose_signature_before, + decompose_signature_after, + "transactional decomposition signature", +) + +decompose_guard_before = ''' if root_row["status"] != "triage": + return None + tenant = root_row["tenant"] +''' +decompose_guard_after = ''' if root_row["status"] != "triage": + return None + if require_no_runs and conn.execute( + "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", + (task_id,), + ).fetchone() is not None: + return None + tenant = root_row["tenant"] +''' +db = replace_once( + db, + decompose_guard_before, + decompose_guard_after, + "transactional decomposition execution-history guard", +) +db_path.write_text(db, encoding="utf-8") + decompose_path = source_root / "hermes_cli/kanban_decompose.py" decompose = decompose_path.read_text(encoding="utf-8") @@ -87,17 +187,20 @@ decompose = replace_once( "auto-decompose preflight guard", ) -single_before = ''' with kb.connect_closing() as conn: - ok = kb.specify_triage_task( -''' -single_after = ''' if automatic and _has_execution_history(task_id): - return DecomposeOutcome( - task_id, - False, - "task gained execution history and requires deliberate manual triage", +single_before = ''' author=audit_author, ) - with kb.connect_closing() as conn: - ok = kb.specify_triage_task( + if not ok: +''' +single_after = ''' author=audit_author, + require_no_runs=automatic, + ) + if not ok: + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) ''' decompose = replace_once( decompose, @@ -106,25 +209,43 @@ decompose = replace_once( "auto-decompose single-task commit guard", ) -fanout_before = ''' try: - with kb.connect_closing() as conn: - child_ids = kb.decompose_triage_task( +fanout_before = ''' author=audit_author, + auto_promote=auto_promote, + ) ''' -fanout_after = ''' if automatic and _has_execution_history(task_id): - return DecomposeOutcome( - task_id, - False, - "task gained execution history and requires deliberate manual triage", - ) - try: - with kb.connect_closing() as conn: - child_ids = kb.decompose_triage_task( +fanout_after = ''' author=audit_author, + auto_promote=auto_promote, + require_no_runs=automatic, + ) ''' decompose = replace_once( decompose, fanout_before, fanout_after, - "auto-decompose fanout commit guard", + "auto-decompose transactional fanout guard", +) + +fanout_outcome_before = ''' if child_ids is None: + return DecomposeOutcome( + task_id, False, "task moved out of triage before decomposition", + ) +''' +fanout_outcome_after = ''' if child_ids is None: + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) + return DecomposeOutcome( + task_id, False, "task moved out of triage before decomposition", + ) +''' +decompose = replace_once( + decompose, + fanout_outcome_before, + fanout_outcome_after, + "auto-decompose fanout rejection reason", ) decompose_path.write_text(decompose, encoding="utf-8") diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index c26f26ca..7fa30e34 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -4,12 +4,14 @@ from __future__ import annotations import concurrent.futures +import hashlib import json import os import re import selectors import signal import sqlite3 +import stat import subprocess import sys import threading @@ -43,12 +45,17 @@ HEARTBEAT_SECONDS = 20 PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60 KANBAN_STORAGE_ATTEMPTS = 5 +ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60 +ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60 +ARTIFACT_RETENTION_COUNT = 256 +ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024 PROVIDER_HEALTH_PATHS = { "codex": DATA_ROOT / "provider-health/codex.json", "claude": DATA_ROOT / "provider-health/claude.json", } WORKTREE_LOCK = threading.Lock() BOARD_CORRUPTION_ERRORS: dict[str, str] = {} +LAST_ARTIFACT_GC = 0.0 CAPACITY_PATTERN = re.compile( r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", re.I, @@ -127,22 +134,65 @@ class TerminalFinalizationPending(RuntimeError): """An accepted worker result is durable but not committed to Kanban yet.""" +@dataclass(frozen=True) +class TerminalIdentity: + """Replay authority derived only from a journal's directory and filename.""" + + board: str + task_id: str + run_id: int + state: str + + def utc_now() -> str: return datetime.now(timezone.utc).isoformat() def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: - """Durably replace a small non-secret state document.""" - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") - temporary.chmod(mode) - os.replace(temporary, path) + """Durably replace a small state document without following temp symlinks.""" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(path.parent, 0o700, follow_symlinks=False) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = None + try: + descriptor = os.open(temporary, flags, mode) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + descriptor = None + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, mode, follow_symlinks=False) + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + if descriptor is not None: + os.close(descriptor) + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _fsync_directory(directory: Path) -> None: + """Persist directory-entry changes after an atomic rename or quarantine.""" + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(directory, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) def load_json(path: Path) -> dict[str, Any]: try: - value = json.loads(path.read_text(encoding="utf-8")) + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "r", encoding="utf-8") as stream: + value = json.load(stream) except (OSError, json.JSONDecodeError): return {} return value if isinstance(value, dict) else {} @@ -313,10 +363,80 @@ def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path: ) -def _terminal_path(state_file: Path, run_id: Any) -> Path: - """Return the replay journal path for an accepted terminal response.""" +def _terminal_path( + state_file: Path, + run_id: Any, + state: str = "pending", +) -> Path: + """Return the identity-bound replay path for an accepted terminal response.""" + if state not in {"pending", "committed"}: + raise ValueError(f"invalid terminal journal state: {state}") safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) - return state_file.with_name(f"{state_file.stem}.run-{safe_run}.terminal.json") + return state_file.with_name( + f"{state_file.stem}.run-{safe_run}.terminal.{state}.json" + ) + + +_TERMINAL_NAME = re.compile( + r"^(?P[a-zA-Z0-9_.-]+)\.run-(?P[0-9]+)\." + r"terminal\.(?Ppending|committed)\.json$" +) +_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$") + + +def _terminal_identity(path: Path) -> TerminalIdentity | None: + """Parse replay authority lexically before opening a journal or board.""" + try: + relative = path.relative_to(STATE_ROOT) + except ValueError: + return None + if len(relative.parts) != 2: + return None + board, filename = relative.parts + match = _TERMINAL_NAME.fullmatch(filename) + try: + board_stat = path.parent.stat(follow_symlinks=False) + except OSError: + return None + if ( + not match + or not stat.S_ISDIR(board_stat.st_mode) + or not _SAFE_BOARD.fullmatch(board) + or board in {".", ".."} + or match.group("task") in {".", ".."} + ): + return None + return TerminalIdentity( + board=board, + task_id=match.group("task"), + run_id=int(match.group("run")), + state=match.group("state"), + ) + + +def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]: + """Read through the already-validated board directory without symlink hops.""" + if _terminal_identity(path) != identity: + return {} + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + directory = None + descriptor = None + try: + directory = os.open(path.parent, directory_flags) + descriptor = os.open(path.name, file_flags, dir_fd=directory) + with os.fdopen(descriptor, "r", encoding="utf-8") as stream: + descriptor = None + value = json.load(stream) + except (OSError, json.JSONDecodeError): + return {} + finally: + if descriptor is not None: + os.close(descriptor) + if directory is not None: + os.close(directory) + return value if isinstance(value, dict) else {} def _persist_candidate( @@ -381,7 +501,10 @@ def _write_terminal_record( return path, record -def _terminal_record_valid(record: dict[str, Any]) -> bool: +def _terminal_record_valid( + record: dict[str, Any], + identity: TerminalIdentity | None = None, +) -> bool: """Reject malformed or non-terminal replay journals without side effects.""" if not isinstance(record, dict): return False @@ -399,10 +522,18 @@ def _terminal_record_valid(record: dict[str, Any]) -> bool: structured = json.loads(record["result"]) except (TypeError, json.JSONDecodeError): return False - return ( + valid = ( isinstance(structured, dict) and structured.get("status") == "completed" and not structured.get("blockers") + and record.get("kanban_state") in {"pending", "committed"} + ) + if not valid or identity is None: + return valid + return ( + record.get("board") == identity.board + and record.get("task_id") == identity.task_id + and record.get("expected_run_id") == identity.run_id ) @@ -945,21 +1076,142 @@ def _board_call( raise last_storage_error +def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: str) -> Path: + """Move an unusable journal aside with a deterministic, private name.""" + board_dir = path.parent + try: + board_stat = board_dir.stat(follow_symlinks=False) + except OSError: + board_stat = None + if board_stat is None or not stat.S_ISDIR(board_stat.st_mode): + print( + f"refused terminal journal path through unsafe board directory: {path}", + file=sys.stderr, + flush=True, + ) + return path + quarantine = board_dir / "quarantine" + quarantine.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(quarantine, 0o700, follow_symlinks=False) + try: + source_stat = path.stat(follow_symlinks=False) + regular = stat.S_ISREG(source_stat.st_mode) + except OSError: + regular = False + try: + if not regular: + raise OSError("journal is not a regular file") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as stream: + fingerprint = hashlib.sha256(stream.read()).hexdigest()[:16] + except OSError: + fingerprint = hashlib.sha256(path.name.encode("utf-8")).hexdigest()[:16] + safe_reason = re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" + destination = quarantine / f"{path.name}.{safe_reason}.{fingerprint}.quarantine" + source_exists = path.exists() or path.is_symlink() + if source_exists and regular and not destination.exists(): + os.chmod(path, 0o600, follow_symlinks=False) + os.replace(path, destination) + os.chmod(destination, 0o600, follow_symlinks=False) + _fsync_directory(board_dir) + _fsync_directory(quarantine) + elif source_exists: + path.unlink() + _fsync_directory(board_dir) + if not destination.exists(): + atomic_json( + destination, + { + "original_name": path.name, + "reason": reason, + "recorded_at": utc_now(), + }, + ) + print( + "quarantined terminal journal " + f"{path.name}: {reason}; identity={identity or 'unparseable'}", + file=sys.stderr, + flush=True, + ) + return destination + + +def _recover_quarantined_run( + kanban_db: Any, + identity: TerminalIdentity | None, + reason: str, +) -> bool: + """Make an exact external run retryable after its journal is quarantined.""" + if identity is None: + return False + + def operation(conn: Any) -> bool: + task = kanban_db.get_task(conn, identity.task_id) + if task is None: + return False + if ( + str(_task_value(task, "status", "")) != "running" + or _task_value(task, "current_run_id", None) != identity.run_id + or not _external(task) + ): + return False + return bool( + kanban_db.reclaim_task( + conn, + identity.task_id, + reason=f"terminal journal quarantined ({reason}); exact run may retry", + ) + ) + + try: + recovered = bool(_board_call(kanban_db, identity.board, operation)) + except Exception as error: + _record_board_access_error(identity.board, error) + return False + if recovered: + print( + f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} " + f"after terminal journal quarantine: {reason}", + file=sys.stderr, + flush=True, + ) + return recovered + + +def _commit_terminal_file(path: Path, identity: TerminalIdentity, document: dict[str, Any]) -> Path: + """Persist committed state, then atomically retire a pending journal.""" + committed = _terminal_path( + state_path(identity.board, identity.task_id), + identity.run_id, + "committed", + ) + document["kanban_state"] = "committed" + document["committed_at"] = utc_now() + atomic_json(path, document) + os.replace(path, committed) + os.chmod(committed, 0o600, follow_symlinks=False) + _fsync_directory(committed.parent) + return committed + + def _finalize_terminal_record( kanban_db: Any, path: Path, - record: dict[str, Any] | None = None, + _record: dict[str, Any] | None = None, ) -> str: """Commit one exact-run terminal journal, or leave it safely pending.""" - document = dict(record or load_json(path)) + identity = _terminal_identity(path) + if identity is None or identity.state != "pending": + return "invalid" + document = _load_terminal_json(path, identity) if not _terminal_record_valid(document): return "invalid" - board = str(document["board"]) - task_id = str(document["task_id"]) - expected_run_id = int(document["expected_run_id"]) + if not _terminal_record_valid(document, identity): + return "foreign" def operation(conn: Any) -> str: - task = kanban_db.get_task(conn, task_id) + task = kanban_db.get_task(conn, identity.task_id) if task is None: return "stale" status = str(_task_value(task, "status", "")) @@ -971,24 +1223,22 @@ def _finalize_terminal_record( ) if ( status != "running" - or _task_value(task, "current_run_id", None) != expected_run_id + or _task_value(task, "current_run_id", None) != identity.run_id ): return "stale" completed = kanban_db.complete_task( conn, - task_id, + identity.task_id, result=document["result"], summary=document["summary"], metadata=document["metadata"], - expected_run_id=expected_run_id, + expected_run_id=identity.run_id, ) return "committed" if completed else "pending" - outcome = str(_board_call(kanban_db, board, operation)) + outcome = str(_board_call(kanban_db, identity.board, operation)) if outcome == "committed": - document["kanban_state"] = "committed" - document["committed_at"] = utc_now() - atomic_json(path, document) + _commit_terminal_file(path, identity, document) return outcome @@ -997,18 +1247,30 @@ def recover_pending_finalizations() -> int: from hermes_cli import kanban_db recovered = 0 - for path in sorted(STATE_ROOT.glob("*/*.terminal.json")): - record = load_json(path) - if record.get("kanban_state") == "committed": + for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")): + identity = _terminal_identity(path) + record = _load_terminal_json(path, identity) if identity is not None else {} + if identity is None: + _quarantine_terminal(path, None, "malformed-name") + continue + if not _terminal_record_valid(record): + _quarantine_terminal(path, identity, "malformed-payload") + _recover_quarantined_run(kanban_db, identity, "malformed-payload") + continue + if not _terminal_record_valid(record, identity): + _quarantine_terminal(path, identity, "foreign-identity") + _recover_quarantined_run(kanban_db, identity, "foreign-identity") continue try: outcome = _finalize_terminal_record(kanban_db, path, record) except Exception as error: - board = str(record.get("board") or "unknown") - _record_board_access_error(board, error) + _record_board_access_error(identity.board, error) continue if outcome == "committed": recovered += 1 + elif outcome in {"invalid", "foreign", "stale"}: + _quarantine_terminal(path, identity, outcome) + _recover_quarantined_run(kanban_db, identity, outcome) return recovered @@ -1016,22 +1278,101 @@ def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool: """Keep an exact run claimed while its accepted result awaits replay.""" if not isinstance(run_id, int): return False - path = _terminal_path(state_path(board, task_id), run_id) + path = _terminal_path(state_path(board, task_id), run_id, "pending") try: path.stat() except FileNotFoundError: return False except OSError: - return True - record = load_json(path) - if not _terminal_record_valid(record): - return True - identity = ( - record.get("board") == board - and record.get("task_id") == task_id - and record.get("expected_run_id") == run_id + return False + identity = _terminal_identity(path) + if identity is None: + return False + record = _load_terminal_json(path, identity) + return _terminal_record_valid(record, identity) + + +def _artifact_gc_candidates(board_dir: Path) -> list[Path]: + """List only bounded artifact classes; pending journals are excluded.""" + candidates: list[Path] = [] + patterns = ( + "*.provider-*.result.json", + "*.candidate-*.json", + "*.terminal.committed.json", ) - return not identity or record.get("kanban_state") != "committed" + for pattern in patterns: + candidates.extend(board_dir.glob(pattern)) + quarantine = board_dir / "quarantine" + if quarantine.is_dir(): + candidates.extend(quarantine.glob("*.quarantine")) + return candidates + + +def gc_lane_artifacts( + *, + now: float | None = None, + max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS, + max_count: int = ARTIFACT_RETENTION_COUNT, + max_bytes: int = ARTIFACT_RETENTION_BYTES, +) -> int: + """Bound non-pending lane artifacts by age, count, and bytes per board.""" + current = time.time() if now is None else now + removed = 0 + for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else (): + if not board_dir.is_dir() or board_dir.is_symlink(): + continue + board_removed = 0 + entries: list[tuple[Path, os.stat_result]] = [] + for path in _artifact_gc_candidates(board_dir): + try: + file_stat = path.stat(follow_symlinks=False) + except OSError: + continue + if not stat.S_ISREG(file_stat.st_mode): + continue + if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds: + path.unlink() + removed += 1 + board_removed += 1 + else: + entries.append((path, file_stat)) + retained_count = 0 + retained_bytes = 0 + for path, file_stat in sorted( + entries, key=lambda item: item[1].st_mtime, reverse=True + ): + exceeds_count = max_count >= 0 and retained_count >= max_count + exceeds_bytes = ( + max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes + ) + if exceeds_count or exceeds_bytes: + path.unlink() + removed += 1 + board_removed += 1 + continue + retained_count += 1 + retained_bytes += file_stat.st_size + if board_removed: + _fsync_directory(board_dir) + return removed + + +def maybe_gc_lane_artifacts(*, now: float | None = None) -> int: + """Run artifact retention on a bounded cadence, not every dispatcher tick.""" + global LAST_ARTIFACT_GC + current = time.time() if now is None else now + if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS: + return 0 + LAST_ARTIFACT_GC = current + try: + return gc_lane_artifacts(now=current) + except OSError as error: + print( + f"lane artifact retention deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + return 0 def execute_claim(board: str, task_id: str) -> None: @@ -1176,6 +1517,16 @@ def execute_claim(board: str, task_id: str) -> None: heartbeat, remaining, ) + candidate_file = None + if result.structured: + candidate_file = _persist_candidate( + state, + state_file, + dict(result.structured), + route=route, + returncode=result.returncode, + goal_turn=goal_turn, + ) if result.capacity_failure: unavailable_provider = route.provider retry_context = ( @@ -1206,6 +1557,15 @@ def execute_claim(board: str, task_id: str) -> None: max(1, int(deadline - time.monotonic())), ) route = fallback + if result.structured: + candidate_file = _persist_candidate( + state, + state_file, + dict(result.structured), + route=route, + returncode=result.returncode, + goal_turn=goal_turn, + ) structured = result.structured if structured: structured = dict(structured) @@ -1213,16 +1573,6 @@ def execute_claim(board: str, task_id: str) -> None: workspace, structured.get("artifacts"), ) - candidate_file = _persist_candidate( - state, - state_file, - structured, - route=route, - returncode=result.returncode, - goal_turn=goal_turn, - ) - else: - candidate_file = None metadata = { "executor": "direct-cli-lane", "provider": route.provider, @@ -1281,8 +1631,9 @@ def execute_claim(board: str, task_id: str) -> None: and result.returncode == 0 and completion_problem is None ): - terminal_file = _terminal_path(state_file, run_id) - metadata["terminal_record"] = str(terminal_file) + metadata["terminal_record"] = str( + _terminal_path(state_file, run_id, "committed") + ) terminal_file, terminal_record = _write_terminal_record( state_file, board=board, @@ -1292,11 +1643,17 @@ def execute_claim(board: str, task_id: str) -> None: summary=str(structured.get("summary") or "Completed"), metadata=metadata, ) - outcome = _finalize_terminal_record( - kanban_db, - terminal_file, - terminal_record, - ) + try: + outcome = _finalize_terminal_record( + kanban_db, + terminal_file, + terminal_record, + ) + except Exception as error: + raise TerminalFinalizationPending( + "accepted worker result is durably journaled; " + f"Kanban finalization raised {type(error).__name__}" + ) from error if outcome != "committed": raise TerminalFinalizationPending( "accepted worker result is durably journaled but " @@ -1537,6 +1894,7 @@ def main() -> int: print(f"worker future failed: {error}", file=sys.stderr, flush=True) del futures[future] recover_pending_finalizations() + maybe_gc_lane_artifacts() active = set(futures.values()) try: newly_claimed = claim_ready(active, workers - len(futures)) diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index b3f53f65..75390d25 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -727,7 +727,7 @@ def test_accepted_result_survives_failed_finalization_and_replays_exact_run( lanes.execute_claim("cassandra", "t_terminal") candidates = list((state_root / "cassandra").glob("*.candidate-*.json")) - terminals = list((state_root / "cassandra").glob("*.terminal.json")) + terminals = list((state_root / "cassandra").glob("*.terminal.pending.json")) assert len(candidates) == 1 assert json.loads(candidates[0].read_text())["structured"] == structured assert len(terminals) == 1 @@ -739,7 +739,12 @@ def test_accepted_result_survives_failed_finalization_and_replays_exact_run( assert lanes.recover_pending_finalizations() == 1 assert task.status == "done" assert len(completions) == 2 - terminal = json.loads(terminals[0].read_text()) + assert not terminals[0].exists() + committed = list( + (state_root / "cassandra").glob("*.terminal.committed.json") + ) + assert len(committed) == 1 + terminal = json.loads(committed[0].read_text()) assert terminal["kanban_state"] == "committed" assert completions[-1]["result"] == terminal["result"] @@ -842,7 +847,334 @@ def test_terminal_replay_never_crosses_into_a_replacement_run( assert lanes.recover_pending_finalizations() == 0 assert completions == [] - assert json.loads(terminal_path.read_text())["kanban_state"] == "pending" + assert not terminal_path.exists() + quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + + +def _completed_result(summary: str = "done") -> dict: + return { + "status": "completed", + "summary": summary, + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + } + + +def test_complete_exception_after_journal_is_replayable_and_never_blocks( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_crash", + status="running", + result=None, + current_run_id=51, + assignee="cli-auto", + max_runtime_seconds=60, + ) + completion_raises = {"value": True} + blocks = [] + comments = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + if completion_raises["value"]: + raise RuntimeError("crash between journal and commit") + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish safely.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "", _completed_result("crash-safe result"), False + ), + ) + + lanes.execute_claim("cassandra", "t_crash") + + pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) + assert len(pending) == 1 + assert blocks == [] + assert any("RuntimeError" in body for body in comments) + + completion_raises["value"] = False + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert not pending[0].exists() + + +def test_terminal_payload_cannot_select_a_different_board( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3) + lanes.atomic_json( + path, + { + "board": "beta", + "task_id": "t_beta", + "expected_run_id": 9, + "result": json.dumps(_completed_result()), + "summary": "forged", + "metadata": {}, + "kanban_state": "pending", + }, + ) + alpha_task = SimpleNamespace( + id="t_alpha", status="running", current_run_id=3, assignee="cli-auto" + ) + opened = [] + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (opened.append(board) or Connection()), + get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None, + reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + + assert opened == ["alpha"] + assert reclaimed == ["t_alpha"] + assert not path.exists() + quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 + + +def test_terminal_finalize_rereads_journal_after_a_path_swap( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("alpha", "t_alpha") + path, safe_record = lanes._write_terminal_record( + state_file, + board="alpha", + task_id="t_alpha", + run_id=4, + structured=_completed_result("safe"), + summary="safe", + metadata={}, + ) + forged = dict(safe_record) + forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7}) + lanes.atomic_json(path, forged) + opened = [] + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")), + ) + + assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign" + assert opened == [] + + +@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"']) +def test_malformed_exact_run_journal_is_quarantined_and_reclaimed( + tmp_path: Path, + monkeypatch, + payload: bytes, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12) + path.parent.mkdir(parents=True) + path.write_bytes(payload) + task = SimpleNamespace( + id="t_partial", status="running", current_run_id=12, assignee="cli-auto" + ) + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + + assert reclaimed == [True] + assert not path.exists() + assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False + quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 + + +def test_recovery_does_not_parse_committed_journals_every_tick( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + committed = lanes._terminal_path( + lanes.state_path("cassandra", "t_done"), 1, "committed" + ) + lanes.atomic_json(committed, {"kanban_state": "committed"}) + reads = [] + monkeypatch.setattr(lanes, "load_json", lambda path: (reads.append(path) or {})) + monkeypatch.setitem( + sys.modules, + "hermes_cli", + SimpleNamespace(kanban_db=SimpleNamespace()), + ) + + assert lanes.recover_pending_finalizations() == 0 + assert reads == [] + + +def test_atomic_json_fsyncs_file_and_directory_and_uses_private_mode( + tmp_path: Path, + monkeypatch, +): + calls = [] + real_fsync = lanes.os.fsync + monkeypatch.setattr( + lanes.os, + "fsync", + lambda descriptor: (calls.append(descriptor), real_fsync(descriptor))[1], + ) + path = tmp_path / "state.json" + + lanes.atomic_json(path, {"safe": True}) + + assert json.loads(path.read_text()) == {"safe": True} + assert path.stat().st_mode & 0o777 == 0o600 + assert len(calls) >= 2 + + +def test_atomic_json_refuses_a_precreated_temp_symlink( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "state.json" + victim = tmp_path / "credential" + victim.write_text("do not overwrite", encoding="utf-8") + temporary = tmp_path / ".state.json.fixed.tmp" + temporary.symlink_to(victim) + monkeypatch.setattr(lanes.uuid, "uuid4", lambda: SimpleNamespace(hex="fixed")) + + with pytest.raises(FileExistsError): + lanes.atomic_json(path, {"unsafe": True}) + + assert victim.read_text(encoding="utf-8") == "do not overwrite" + assert not path.exists() + + +def test_artifact_gc_prunes_by_age_without_touching_pending_journals( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + quarantine = board / "quarantine" + quarantine.mkdir() + old_artifacts = [ + board / "t.run-1.candidate-1.json", + board / "t.run-1.provider-1.result.json", + board / "t.run-1.terminal.committed.json", + quarantine / "t.invalid.1234.quarantine", + ] + for artifact in old_artifacts: + artifact.write_text("{}", encoding="utf-8") + pending = board / "t.run-1.terminal.pending.json" + pending.write_text("{}", encoding="utf-8") + old_time = 100.0 + for artifact in old_artifacts: + os.utime(artifact, (old_time, old_time)) + os.utime(pending, (old_time, old_time)) + + assert lanes.gc_lane_artifacts( + now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 + ) == len(old_artifacts) + assert not any(artifact.exists() for artifact in old_artifacts) + assert pending.exists() + + +def test_artifact_gc_prunes_oldest_by_count_and_total_bytes( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + files = [] + for sequence in range(3): + path = board / f"t.run-1.provider-{sequence}.result.json" + path.write_text("x" * 40, encoding="utf-8") + os.utime(path, (100 + sequence, 100 + sequence)) + files.append(path) + + assert lanes.gc_lane_artifacts( + now=200.0, max_age_seconds=1000, max_count=2, max_bytes=45 + ) == 2 + assert [path.exists() for path in files] == [False, False, True] + + +def test_artifact_gc_is_interval_gated( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0) + calls = [] + monkeypatch.setattr( + lanes, "gc_lane_artifacts", lambda **kwargs: (calls.append(kwargs) or 0) + ) + + assert lanes.maybe_gc_lane_artifacts(now=1000.0) == 0 + assert lanes.maybe_gc_lane_artifacts(now=1001.0) == 0 + assert len(calls) == 1 @pytest.mark.parametrize( @@ -936,7 +1268,7 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts( block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") monkeypatch.setattr( lanes, "select_route", @@ -1004,11 +1336,7 @@ def test_goal_card_continues_after_local_judge_rejects_progress( block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") claude_low = lanes.Route( "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () ) @@ -1087,7 +1415,9 @@ def test_goal_card_continues_after_local_judge_rejects_progress( assert route_calls[2][1]["exclude_provider"] == "claude" assert "prior rejected reports" in judge_contexts[1] assert "commit, push, and remote verification are missing" in judge_contexts[1] - candidates = sorted(tmp_path.glob("state.run-12.candidate-*.json")) + candidates = sorted( + (lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json") + ) assert len(candidates) == 2 assert json.loads(candidates[0].read_text())["structured"]["summary"] == ( "Focused tests passed." @@ -1098,6 +1428,87 @@ def test_goal_card_continues_after_local_judge_rejects_progress( assert reports == [] +def test_capacity_fallback_preserves_first_claude_structured_response( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_fallback", + status="running", + result=None, + current_run_id=14, + assignee="cli-auto", + max_runtime_seconds=60, + ) + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Complete with a fallback if needed.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda *_args: None, + complete_task=lambda _conn, _task_id, **kwargs: ( + setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True + ), + block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + claude = lanes.Route( + "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, () + ) + codex = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda _prompt, assignee, **_kwargs: codex + if assignee == "cli-codex-high" + else claude, + ) + reports = [ + lanes.ProcessResult( + 1, + "subscription capacity exhausted", + { + **_completed_result("Claude preserved evidence"), + "status": "incomplete", + "blockers": ["subscription capacity exhausted"], + }, + True, + ), + lanes.ProcessResult(0, "", _completed_result("Codex completed"), False), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + + lanes.execute_claim("cassandra", "t_fallback") + + candidates = sorted( + (lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json") + ) + assert len(candidates) == 2 + first, second = [json.loads(path.read_text()) for path in candidates] + assert (first["provider"], first["structured"]["summary"]) == ( + "claude", + "Claude preserved evidence", + ) + assert (second["provider"], second["structured"]["summary"]) == ( + "codex", + "Codex completed", + ) + assert task.status == "done" + + def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") calls = [] @@ -1143,11 +1554,15 @@ def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): task = SimpleNamespace( id="t_resume", + status="running", + result=None, current_run_id=9, assignee="cli-auto", max_runtime_seconds=60, ) - state_file = tmp_path / "state.json" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_resume") + state_file.parent.mkdir(parents=True) state_file.write_text( json.dumps({"current_route": {"provider": "claude"}}), encoding="utf-8", @@ -1173,7 +1588,6 @@ def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: P block_task=lambda *_args, **_kwargs: None, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) monkeypatch.setattr( lanes, "select_route", From c20bc263e0916b01fda6f932fcc5717194b0e6ca Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:55:46 -0300 Subject: [PATCH 05/46] hermes: close terminal replay edge cases --- .../hermes-execution-safety-regression.py | 62 ++++ dockerfiles/patch-hermes-execution-safety.py | 108 +++++- services/hermes/scripts/cli_lane_runner.py | 259 +++++++++++---- testing/tests/test_hermes_cli_lanes.py | 311 +++++++++++++++++- 4 files changed, 669 insertions(+), 71 deletions(-) diff --git a/dockerfiles/hermes-execution-safety-regression.py b/dockerfiles/hermes-execution-safety-regression.py index a632e51b..a77ca92d 100644 --- a/dockerfiles/hermes-execution-safety-regression.py +++ b/dockerfiles/hermes-execution-safety-regression.py @@ -146,6 +146,68 @@ class AutomaticDecompositionSafetyTests(unittest.TestCase): self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "ready") + def test_latest_ended_run_can_replay_a_durable_completion(self) -> None: + task_id = kanban_db.create_task(self.connection, title="journaled result") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="legacy post-journal failure", + kind="capability", + expected_run_id=run_id, + ) + ) + + self.assertTrue( + kanban_db.complete_task( + self.connection, + task_id, + result="durable terminal result", + summary="durable terminal result", + replay_ended_run_id=run_id, + ) + ) + self.assertEqual(self._status(task_id), "done") + + def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None: + task_id = kanban_db.create_task(self.connection, title="replacement guard") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + old_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="first run", + kind="capability", + expected_run_id=old_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertNotEqual(old_run, replacement_run) + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="replacement run", + kind="capability", + expected_run_id=replacement_run, + ) + ) + + self.assertFalse( + kanban_db.complete_task( + self.connection, + task_id, + result="stale result", + replay_ended_run_id=old_run, + ) + ) + self.assertIn(self._status(task_id), {"blocked", "triage"}) + def test_fresh_triage_task_still_auto_promotes(self) -> None: task_id = kanban_db.create_task( self.connection, diff --git a/dockerfiles/patch-hermes-execution-safety.py b/dockerfiles/patch-hermes-execution-safety.py index 621d508d..6abd6f26 100644 --- a/dockerfiles/patch-hermes-execution-safety.py +++ b/dockerfiles/patch-hermes-execution-safety.py @@ -1,4 +1,4 @@ -"""Patch upstream Hermes auto-decomposition to respect execution history.""" +"""Patch upstream Hermes exact-run completion and decomposition safety.""" import os from pathlib import Path @@ -18,6 +18,112 @@ source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) db_path = source_root / "hermes_cli/kanban_db.py" db = db_path.read_text(encoding="utf-8") +complete_signature_before = '''def complete_task( + conn: sqlite3.Connection, + task_id: str, + *, + result: Optional[str] = None, + summary: Optional[str] = None, + metadata: Optional[dict] = None, + created_cards: Optional[Iterable[str]] = None, + expected_run_id: Optional[int] = None, +) -> bool: +''' +complete_signature_after = '''def complete_task( + conn: sqlite3.Connection, + task_id: str, + *, + result: Optional[str] = None, + summary: Optional[str] = None, + metadata: Optional[dict] = None, + created_cards: Optional[Iterable[str]] = None, + expected_run_id: Optional[int] = None, + replay_ended_run_id: Optional[int] = None, +) -> bool: +''' +db = replace_once( + db, + complete_signature_before, + complete_signature_after, + "exact ended-run completion signature", +) + +complete_branch_before = ''' with write_txn(conn): + if expected_run_id is None: + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('running', 'ready', 'blocked', 'scheduled') + """, + (result, now, task_id), + ) + else: +''' +complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the + # accepted run. Verify that it is still the latest run in this same + # transaction; a replacement attempt must make the replay fail closed. + with write_txn(conn): + if expected_run_id is not None and replay_ended_run_id is not None: + raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive") + if replay_ended_run_id is not None: + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('ready', 'blocked') + AND current_run_id IS NULL + AND ? = ( + SELECT id FROM task_runs + WHERE task_id = ? + ORDER BY id DESC + LIMIT 1 + ) + """, + (result, now, task_id, int(replay_ended_run_id), task_id), + ) + elif expected_run_id is None: + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('running', 'ready', 'blocked', 'scheduled') + """, + (result, now, task_id), + ) + else: +''' +db = replace_once( + db, + complete_branch_before, + complete_branch_after, + "transactional exact ended-run completion", +) + specify_signature_before = '''def specify_triage_task( conn: sqlite3.Connection, task_id: str, diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 7fa30e34..07e90e48 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -522,10 +522,27 @@ def _terminal_record_valid( structured = json.loads(record["result"]) except (TypeError, json.JSONDecodeError): return False + if not isinstance(structured, dict): + return False + required = set(RESULT_SCHEMA["required"]) + if set(structured) != required: + return False + if ( + type(structured["status"]) is not str + or structured["status"] not in cli_lane_goal.RESULT_STATUSES + or type(structured["summary"]) is not str + or not structured["summary"].strip() + ): + return False + for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + value = structured[key] + if type(value) is not list or any(type(item) is not str for item in value): + return False valid = ( - isinstance(structured, dict) - and structured.get("status") == "completed" - and not structured.get("blockers") + structured["status"] == "completed" + and structured["blockers"] == [] + and cli_lane_goal.unfinished_result_reason(structured) is None + and record["summary"] == structured["summary"] and record.get("kanban_state") in {"pending", "committed"} ) if not valid or identity is None: @@ -1077,7 +1094,7 @@ def _board_call( def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: str) -> Path: - """Move an unusable journal aside with a deterministic, private name.""" + """Copy an unusable journal to a private file, then retire its exact name.""" board_dir = path.parent try: board_stat = board_dir.stat(follow_symlinks=False) @@ -1090,44 +1107,135 @@ def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: flush=True, ) return path - quarantine = board_dir / "quarantine" - quarantine.mkdir(parents=True, exist_ok=True, mode=0o700) - os.chmod(quarantine, 0o700, follow_symlinks=False) + board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + board_flags |= getattr(os, "O_NOFOLLOW", 0) + file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + create_flags |= getattr(os, "O_NOFOLLOW", 0) + board_descriptor = None + quarantine_descriptor = None + source_stat = None + destination = path + retired = False try: - source_stat = path.stat(follow_symlinks=False) - regular = stat.S_ISREG(source_stat.st_mode) - except OSError: - regular = False - try: - if not regular: - raise OSError("journal is not a regular file") - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(path, flags) - with os.fdopen(descriptor, "rb") as stream: - fingerprint = hashlib.sha256(stream.read()).hexdigest()[:16] - except OSError: - fingerprint = hashlib.sha256(path.name.encode("utf-8")).hexdigest()[:16] - safe_reason = re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" - destination = quarantine / f"{path.name}.{safe_reason}.{fingerprint}.quarantine" - source_exists = path.exists() or path.is_symlink() - if source_exists and regular and not destination.exists(): - os.chmod(path, 0o600, follow_symlinks=False) - os.replace(path, destination) - os.chmod(destination, 0o600, follow_symlinks=False) - _fsync_directory(board_dir) - _fsync_directory(quarantine) - elif source_exists: - path.unlink() - _fsync_directory(board_dir) - if not destination.exists(): - atomic_json( - destination, - { - "original_name": path.name, - "reason": reason, - "recorded_at": utc_now(), - }, + board_descriptor = os.open(board_dir, board_flags) + source_stat = os.stat( + path.name, + dir_fd=board_descriptor, + follow_symlinks=False, + ) + safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1 + payload: bytes + if safe_source: + source_descriptor = os.open( + path.name, + file_flags, + dir_fd=board_descriptor, ) + with os.fdopen(source_descriptor, "rb") as source: + opened_stat = os.fstat(source.fileno()) + if ( + opened_stat.st_dev != source_stat.st_dev + or opened_stat.st_ino != source_stat.st_ino + or opened_stat.st_nlink != 1 + ): + raise OSError("terminal journal identity changed while opening") + payload = source.read() + else: + payload = ( + json.dumps( + { + "original_name": path.name, + "reason": reason, + "source_kind": "hardlink" + if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1 + else "non-regular", + }, + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + fingerprint = hashlib.sha256(payload).hexdigest()[:16] + safe_reason = ( + re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" + ) + try: + os.mkdir("quarantine", 0o700, dir_fd=board_descriptor) + except FileExistsError: + pass + quarantine_descriptor = os.open( + "quarantine", + board_flags, + dir_fd=board_descriptor, + ) + for sequence in range(32): + name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine" + try: + destination_descriptor = os.open( + name, + create_flags, + 0o600, + dir_fd=quarantine_descriptor, + ) + except FileExistsError: + continue + try: + with os.fdopen(destination_descriptor, "wb") as target: + target.write(payload) + target.flush() + os.fsync(target.fileno()) + target_stat = os.fstat(target.fileno()) + if ( + stat.S_IMODE(target_stat.st_mode) != 0o600 + or target_stat.st_nlink != 1 + ): + raise OSError("quarantine destination is not private and singly linked") + except Exception: + os.unlink(name, dir_fd=quarantine_descriptor) + raise + destination = board_dir / "quarantine" / name + break + else: + raise OSError("could not reserve a unique quarantine destination") + os.fsync(quarantine_descriptor) + current = os.stat( + path.name, + dir_fd=board_descriptor, + follow_symlinks=False, + ) + if ( + current.st_dev != source_stat.st_dev + or current.st_ino != source_stat.st_ino + ): + print( + f"terminal journal changed before retirement; removing new entry: {path}", + file=sys.stderr, + flush=True, + ) + os.unlink(path.name, dir_fd=board_descriptor) + retired = True + os.fsync(board_descriptor) + except OSError as error: + # A bad quarantine target must not make an attacker-controlled pending + # path eligible for replay forever. Retire only the directory entry; + # never chmod or write through the source inode. + if board_descriptor is not None and not retired: + try: + os.unlink(path.name, dir_fd=board_descriptor) + retired = True + os.fsync(board_descriptor) + except OSError: + pass + print( + f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + finally: + if quarantine_descriptor is not None: + os.close(quarantine_descriptor) + if board_descriptor is not None: + os.close(board_descriptor) print( "quarantined terminal journal " f"{path.name}: {reason}; identity={identity or 'unparseable'}", @@ -1137,12 +1245,12 @@ def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: return destination -def _recover_quarantined_run( +def _recover_exact_run( kanban_db: Any, identity: TerminalIdentity | None, reason: str, ) -> bool: - """Make an exact external run retryable after its journal is quarantined.""" + """Make an exact external run retryable after journal recovery fails.""" if identity is None: return False @@ -1160,7 +1268,7 @@ def _recover_quarantined_run( kanban_db.reclaim_task( conn, identity.task_id, - reason=f"terminal journal quarantined ({reason}); exact run may retry", + reason=f"terminal journal recovery failed ({reason}); exact run may retry", ) ) @@ -1172,7 +1280,7 @@ def _recover_quarantined_run( if recovered: print( f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} " - f"after terminal journal quarantine: {reason}", + f"after terminal journal recovery: {reason}", file=sys.stderr, flush=True, ) @@ -1221,10 +1329,16 @@ def _finalize_terminal_record( if str(_task_value(task, "result", "") or "") == document["result"] else "stale" ) - if ( - status != "running" - or _task_value(task, "current_run_id", None) != identity.run_id - ): + current_run_id = _task_value(task, "current_run_id", None) + completion_guard: dict[str, int] + if status == "running" and current_run_id == identity.run_id: + completion_guard = {"expected_run_id": identity.run_id} + elif status in {"ready", "blocked"} and current_run_id is None: + # The journal may have survived an older post-persistence error + # path that ended its run. The patched DB verifies atomically that + # this is still the latest ended run before allowing completion. + completion_guard = {"replay_ended_run_id": identity.run_id} + else: return "stale" completed = kanban_db.complete_task( conn, @@ -1232,7 +1346,7 @@ def _finalize_terminal_record( result=document["result"], summary=document["summary"], metadata=document["metadata"], - expected_run_id=identity.run_id, + **completion_guard, ) return "committed" if completed else "pending" @@ -1255,11 +1369,11 @@ def recover_pending_finalizations() -> int: continue if not _terminal_record_valid(record): _quarantine_terminal(path, identity, "malformed-payload") - _recover_quarantined_run(kanban_db, identity, "malformed-payload") + _recover_exact_run(kanban_db, identity, "malformed-payload") continue if not _terminal_record_valid(record, identity): _quarantine_terminal(path, identity, "foreign-identity") - _recover_quarantined_run(kanban_db, identity, "foreign-identity") + _recover_exact_run(kanban_db, identity, "foreign-identity") continue try: outcome = _finalize_terminal_record(kanban_db, path, record) @@ -1270,7 +1384,7 @@ def recover_pending_finalizations() -> int: recovered += 1 elif outcome in {"invalid", "foreign", "stale"}: _quarantine_terminal(path, identity, outcome) - _recover_quarantined_run(kanban_db, identity, outcome) + _recover_exact_run(kanban_db, identity, outcome) return recovered @@ -1329,6 +1443,9 @@ def gc_lane_artifacts( except OSError: continue if not stat.S_ISREG(file_stat.st_mode): + path.unlink() + removed += 1 + board_removed += 1 continue if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds: path.unlink() @@ -1631,28 +1748,44 @@ def execute_claim(board: str, task_id: str) -> None: and result.returncode == 0 and completion_problem is None ): + terminal_file = _terminal_path(state_file, run_id, "pending") metadata["terminal_record"] = str( _terminal_path(state_file, run_id, "committed") ) - terminal_file, terminal_record = _write_terminal_record( - state_file, - board=board, - task_id=task_id, - run_id=run_id, - structured=structured, - summary=str(structured.get("summary") or "Completed"), - metadata=metadata, - ) try: + terminal_file, terminal_record = _write_terminal_record( + state_file, + board=board, + task_id=task_id, + run_id=run_id, + structured=structured, + summary=str(structured.get("summary") or "Completed"), + metadata=metadata, + ) outcome = _finalize_terminal_record( kanban_db, terminal_file, terminal_record, ) except Exception as error: + identity = _terminal_identity(terminal_file) + replayable = bool( + identity is not None + and _terminal_record_valid( + _load_terminal_json(terminal_file, identity), + identity, + ) + ) + if not replayable: + _recover_exact_run( + kanban_db, + identity, + f"persistence raised {type(error).__name__}", + ) raise TerminalFinalizationPending( - "accepted worker result is durably journaled; " - f"Kanban finalization raised {type(error).__name__}" + "accepted worker result remains outside the generic block path; " + f"terminal replayable={replayable}; persistence/finalization " + f"raised {type(error).__name__}" ) from error if outcome != "committed": raise TerminalFinalizationPending( diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 75390d25..2db9b1ae 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -2,7 +2,9 @@ from __future__ import annotations +import errno import importlib.util +import hashlib import json import os import signal @@ -864,6 +866,55 @@ def _completed_result(summary: str = "done") -> dict: } +def test_terminal_recovery_can_complete_the_exact_latest_ended_run( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_ended") + pending, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_ended", + run_id=17, + structured=_completed_result("accepted before legacy block"), + summary="accepted before legacy block", + metadata={}, + ) + task = SimpleNamespace( + id="t_ended", + status="blocked", + result=None, + current_run_id=None, + assignee="cli-auto", + ) + guards = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + guards.append(kwargs) + task.status = "done" + task.result = kwargs["result"] + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert guards[0]["replay_ended_run_id"] == 17 + assert "expected_run_id" not in guards[0] + assert task.status == "done" + assert not pending.exists() + + def test_complete_exception_after_journal_is_replayable_and_never_blocks( tmp_path: Path, monkeypatch, @@ -937,6 +988,103 @@ def test_complete_exception_after_journal_is_replayable_and_never_blocks( assert not pending[0].exists() +def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_enospc", + status="running", + result=None, + current_run_id=52, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + comments = [] + completions = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + completions.append(kwargs) + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish without losing the result.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "", _completed_result("persisted before ENOSPC"), False + ), + ) + real_replace = lanes.os.replace + real_fsync_directory = lanes._fsync_directory + terminal_replaced = {"value": False} + fail_once = {"value": True} + + def replace_then_mark(source, destination): + real_replace(source, destination) + if str(destination).endswith(".terminal.pending.json"): + terminal_replaced["value"] = True + + def fail_after_terminal_replace(directory): + if terminal_replaced["value"] and fail_once["value"]: + fail_once["value"] = False + raise OSError(errno.ENOSPC, "no space after terminal rename") + real_fsync_directory(directory) + + monkeypatch.setattr(lanes.os, "replace", replace_then_mark) + monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace) + + lanes.execute_claim("cassandra", "t_enospc") + + pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) + assert terminal_replaced["value"] is True + assert fail_once["value"] is False + assert len(pending) == 1 + assert blocks == [] + assert completions == [] + assert task.status == "running" + assert any("terminal replayable=True" in body for body in comments) + + # A restarted runner sees the exact pending journal and completes the run. + assert lanes.recover_pending_finalizations() == 1 + assert len(completions) == 1 + assert task.status == "done" + assert not pending[0].exists() + committed = list((state_root / "cassandra").glob("*.terminal.committed.json")) + assert len(committed) == 1 + + def test_terminal_payload_cannot_select_a_different_board( tmp_path: Path, monkeypatch, @@ -1050,6 +1198,147 @@ def test_malformed_exact_run_journal_is_quarantined_and_reclaimed( assert quarantined[0].stat().st_mode & 0o777 == 0o600 +@pytest.mark.parametrize( + "structured", + [ + {"status": "completed", "blockers": {}}, + {**_completed_result(), "changed_files": "src/a.py"}, + {**_completed_result(), "unexpected": True}, + {**_completed_result(), "blockers": ["work remains"]}, + {**_completed_result("tests are still running")}, + ], +) +def test_terminal_record_requires_exact_completed_result_contract(structured): + record = { + "board": "cassandra", + "task_id": "t_schema", + "expected_run_id": 1, + "result": json.dumps(structured), + "summary": str(structured.get("summary") or "done"), + "metadata": {}, + "kanban_state": "pending", + } + + assert lanes._terminal_record_valid(record) is False + + +def test_minimal_completed_result_with_mapping_blockers_quarantines_without_completion( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_schema"), 6) + lanes.atomic_json( + path, + { + "board": "cassandra", + "task_id": "t_schema", + "expected_run_id": 6, + "result": json.dumps({"status": "completed", "blockers": {}}), + "summary": "done", + "metadata": {}, + "kanban_state": "pending", + }, + ) + task = SimpleNamespace( + id="t_schema", status="running", current_run_id=6, assignee="cli-auto" + ) + completed = [] + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=lambda *_args, **_kwargs: completed.append(True), + reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + assert completed == [] + assert reclaimed == [True] + assert not path.exists() + quarantined = list((path.parent / "quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 + + +def test_quarantine_avoids_symlink_and_mode_collision_destinations( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_collision"), 2) + path.parent.mkdir(parents=True) + path.write_bytes(b"invalid") + quarantine = path.parent / "quarantine" + quarantine.mkdir() + fingerprint = hashlib.sha256(b"invalid").hexdigest()[:16] + base = f"{path.name}.malformed-payload.{fingerprint}" + victim = tmp_path / "victim" + victim.write_text("unchanged", encoding="utf-8") + (quarantine / f"{base}.0.quarantine").symlink_to(victim) + collision = quarantine / f"{base}.1.quarantine" + collision.write_text("attacker collision", encoding="utf-8") + collision.chmod(0o644) + monkeypatch.setattr( + lanes.os, + "chmod", + lambda *_args, **_kwargs: pytest.fail("quarantine must not chmod foreign inodes"), + ) + + destination = lanes._quarantine_terminal( + path, + lanes._terminal_identity(path), + "malformed-payload", + ) + + assert not path.exists() + assert destination.name == f"{base}.2.quarantine" + assert destination.read_bytes() == b"invalid" + assert destination.stat().st_mode & 0o777 == 0o600 + assert victim.read_text(encoding="utf-8") == "unchanged" + assert collision.read_text(encoding="utf-8") == "attacker collision" + assert collision.stat().st_mode & 0o777 == 0o644 + + +def test_hardlinked_terminal_source_is_never_chmodded_or_copied_as_data( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_hardlink"), 5) + path.parent.mkdir(parents=True) + foreign = tmp_path / "foreign" + foreign.write_text("foreign inode contents", encoding="utf-8") + foreign.chmod(0o644) + os.link(foreign, path) + monkeypatch.setattr( + lanes.os, + "chmod", + lambda *_args, **_kwargs: pytest.fail("hardlinked source must not be chmodded"), + ) + + destination = lanes._quarantine_terminal( + path, + lanes._terminal_identity(path), + "malformed-payload", + ) + + assert not path.exists() + assert foreign.read_text(encoding="utf-8") == "foreign inode contents" + assert foreign.stat().st_mode & 0o777 == 0o644 + assert destination.stat().st_mode & 0o777 == 0o600 + diagnostic = json.loads(destination.read_text(encoding="utf-8")) + assert diagnostic["source_kind"] == "hardlink" + assert "foreign inode contents" not in destination.read_text(encoding="utf-8") + + def test_recovery_does_not_parse_committed_journals_every_tick( tmp_path: Path, monkeypatch, @@ -1127,6 +1416,8 @@ def test_artifact_gc_prunes_by_age_without_touching_pending_journals( ] for artifact in old_artifacts: artifact.write_text("{}", encoding="utf-8") + quarantine_symlink = quarantine / "attacker.quarantine" + quarantine_symlink.symlink_to(tmp_path / "missing-target") pending = board / "t.run-1.terminal.pending.json" pending.write_text("{}", encoding="utf-8") old_time = 100.0 @@ -1136,8 +1427,9 @@ def test_artifact_gc_prunes_by_age_without_touching_pending_journals( assert lanes.gc_lane_artifacts( now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 - ) == len(old_artifacts) + ) == len(old_artifacts) + 1 assert not any(artifact.exists() for artifact in old_artifacts) + assert not quarantine_symlink.is_symlink() assert pending.exists() @@ -1189,9 +1481,10 @@ def test_artifact_gc_is_interval_gated( "status": "completed", "summary": "done", "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "findings": [], + "blockers": [], }, False, ), @@ -1205,9 +1498,10 @@ def test_artifact_gc_is_interval_gated( "status": "completed", "summary": "The full test suite is still running.", "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "findings": [], + "blockers": [], }, False, ), @@ -1369,6 +1663,7 @@ def test_goal_card_continues_after_local_judge_rejects_progress( "changed_files": ["src/a.py"], "tests_run": ["pytest focused: passed"], "artifacts": [], + "findings": [], "blockers": [], }, False, @@ -1382,6 +1677,7 @@ def test_goal_card_continues_after_local_judge_rejects_progress( "changed_files": ["src/a.py"], "tests_run": ["pytest full: passed"], "artifacts": [], + "findings": [], "blockers": [], }, False, @@ -1614,6 +1910,7 @@ def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: P "changed_files": [], "tests_run": [], "artifacts": [], + "findings": [], "blockers": [], }, False, From 8c71d55853923965bd39b3084f7a92b57035a63c Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 21:32:03 -0300 Subject: [PATCH 06/46] hermes: harden journal quarantine races --- services/hermes/scripts/cli_lane_runner.py | 336 ++++++++++++++++----- testing/tests/test_hermes_cli_lanes.py | 271 ++++++++++++++++- 2 files changed, 537 insertions(+), 70 deletions(-) diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 07e90e48..30c84daf 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -4,6 +4,8 @@ from __future__ import annotations import concurrent.futures +import ctypes +import errno import hashlib import json import os @@ -49,6 +51,8 @@ ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60 ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60 ARTIFACT_RETENTION_COUNT = 256 ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024 +MAX_TERMINAL_RECORD_BYTES = 1024 * 1024 +QUARANTINE_HASH_BYTES = 64 * 1024 PROVIDER_HEALTH_PATHS = { "codex": DATA_ROOT / "provider-health/codex.json", "claude": DATA_ROOT / "provider-health/claude.json", @@ -193,7 +197,7 @@ def load_json(path: Path) -> dict[str, Any]: descriptor = os.open(path, flags) with os.fdopen(descriptor, "r", encoding="utf-8") as stream: value = json.load(stream) - except (OSError, json.JSONDecodeError): + except (OSError, UnicodeError, json.JSONDecodeError): return {} return value if isinstance(value, dict) else {} @@ -414,8 +418,21 @@ def _terminal_identity(path: Path) -> TerminalIdentity | None: ) +def _read_bounded(descriptor: int, limit: int) -> bytes: + """Read at most ``limit`` bytes from a regular file descriptor.""" + chunks: list[bytes] = [] + remaining = max(0, limit) + while remaining: + chunk = os.read(descriptor, min(64 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]: - """Read through the already-validated board directory without symlink hops.""" + """Read one small, singly-linked journal through its validated directory.""" if _terminal_identity(path) != identity: return {} directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) @@ -426,10 +443,30 @@ def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any try: directory = os.open(path.parent, directory_flags) descriptor = os.open(path.name, file_flags, dir_fd=directory) - with os.fdopen(descriptor, "r", encoding="utf-8") as stream: - descriptor = None - value = json.load(stream) - except (OSError, json.JSONDecodeError): + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_size > MAX_TERMINAL_RECORD_BYTES + ): + return {} + payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1) + if len(payload) > MAX_TERMINAL_RECORD_BYTES: + return {} + current = os.stat( + path.name, + dir_fd=directory, + follow_symlinks=False, + ) + if ( + current.st_dev != opened.st_dev + or current.st_ino != opened.st_ino + or current.st_size != opened.st_size + or len(payload) != opened.st_size + ): + return {} + value = json.loads(payload.decode("utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): return {} finally: if descriptor is not None: @@ -1093,8 +1130,100 @@ def _board_call( raise last_storage_error +def _rename_noreplace( + source: str, + destination: str, + *, + source_dir: int, + destination_dir: int, +) -> None: + """Atomically rename a directory entry without replacing a collision.""" + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None) + if renameat2 is None: + raise OSError(errno.ENOSYS, "renameat2 is unavailable") + renameat2.argtypes = ( + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ) + renameat2.restype = ctypes.c_int + result = renameat2( + source_dir, + ctypes.c_char_p(os.fsencode(source)), + destination_dir, + ctypes.c_char_p(os.fsencode(destination)), + 1, # RENAME_NOREPLACE + ) + if result != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), source, destination) + + +def _retire_terminal_entry( + path: Path, + source_stat: os.stat_result, + *, + board_descriptor: int, + quarantine_descriptor: int, +) -> str: + """Retire only the inode previously inspected; preserve any replacement.""" + try: + current = os.stat( + path.name, + dir_fd=board_descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + return "missing" + if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino: + return "replacement" + identity = hashlib.sha256( + f"{path.name}\0{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode("utf-8") + ).hexdigest()[:32] + for sequence in range(32): + staging = f".retire.{path.name}.{identity}.{sequence}" + try: + _rename_noreplace( + path.name, + staging, + source_dir=board_descriptor, + destination_dir=quarantine_descriptor, + ) + except FileExistsError: + continue + break + else: + return "collision" + staged = os.stat( + staging, + dir_fd=quarantine_descriptor, + follow_symlinks=False, + ) + if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino: + os.unlink(staging, dir_fd=quarantine_descriptor) + os.fsync(quarantine_descriptor) + os.fsync(board_descriptor) + return "retired" + # The path changed after the pre-check but before the rename. Put the + # replacement back without overwriting an even newer entry. + try: + _rename_noreplace( + staging, + path.name, + source_dir=quarantine_descriptor, + destination_dir=board_descriptor, + ) + except FileExistsError: + return "replacement-staged" + os.fsync(quarantine_descriptor) + os.fsync(board_descriptor) + return "replacement" + + def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: str) -> Path: - """Copy an unusable journal to a private file, then retire its exact name.""" + """Record bounded metadata, then retire only the inspected journal inode.""" board_dir = path.parent try: board_stat = board_dir.stat(follow_symlinks=False) @@ -1114,9 +1243,10 @@ def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: create_flags |= getattr(os, "O_NOFOLLOW", 0) board_descriptor = None quarantine_descriptor = None + source_descriptor = None source_stat = None destination = path - retired = False + retirement = "not-attempted" try: board_descriptor = os.open(board_dir, board_flags) source_stat = os.stat( @@ -1125,37 +1255,50 @@ def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: follow_symlinks=False, ) safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1 - payload: bytes + digest = None + hashed_bytes = 0 + hash_complete = False if safe_source: source_descriptor = os.open( path.name, file_flags, dir_fd=board_descriptor, ) - with os.fdopen(source_descriptor, "rb") as source: - opened_stat = os.fstat(source.fileno()) - if ( - opened_stat.st_dev != source_stat.st_dev - or opened_stat.st_ino != source_stat.st_ino - or opened_stat.st_nlink != 1 - ): - raise OSError("terminal journal identity changed while opening") - payload = source.read() - else: - payload = ( - json.dumps( - { - "original_name": path.name, - "reason": reason, - "source_kind": "hardlink" - if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1 - else "non-regular", - }, - sort_keys=True, - ) - + "\n" - ).encode("utf-8") - fingerprint = hashlib.sha256(payload).hexdigest()[:16] + opened_stat = os.fstat(source_descriptor) + if ( + opened_stat.st_dev != source_stat.st_dev + or opened_stat.st_ino != source_stat.st_ino + or opened_stat.st_nlink != 1 + ): + raise OSError("terminal journal identity changed while opening") + # Keep this descriptor open through retirement. The staging entry + # must match the exact fstat-verified inode, not merely a path stat + # captured before a concurrent replacement. + source_stat = opened_stat + prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES) + hashed_bytes = len(prefix) + hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES + digest = hashlib.sha256(prefix).hexdigest() + source_kind = ( + "regular" + if safe_source + else "hardlink" + if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1 + else "non-regular" + ) + diagnostic = { + "hash_complete": hash_complete, + "hashed_bytes": hashed_bytes, + "original_name": path.name, + "reason": reason, + "sha256": digest, + "size": source_stat.st_size, + "source_kind": source_kind, + } + payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8") + fingerprint = ( + (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16] + ) safe_reason = ( re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" ) @@ -1198,47 +1341,46 @@ def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: else: raise OSError("could not reserve a unique quarantine destination") os.fsync(quarantine_descriptor) - current = os.stat( - path.name, - dir_fd=board_descriptor, - follow_symlinks=False, + retirement = _retire_terminal_entry( + path, + source_stat, + board_descriptor=board_descriptor, + quarantine_descriptor=quarantine_descriptor, ) - if ( - current.st_dev != source_stat.st_dev - or current.st_ino != source_stat.st_ino - ): - print( - f"terminal journal changed before retirement; removing new entry: {path}", - file=sys.stderr, - flush=True, - ) - os.unlink(path.name, dir_fd=board_descriptor) - retired = True - os.fsync(board_descriptor) except OSError as error: # A bad quarantine target must not make an attacker-controlled pending - # path eligible for replay forever. Retire only the directory entry; - # never chmod or write through the source inode. - if board_descriptor is not None and not retired: + # path eligible for replay forever. The same identity-safe staging + # protocol works in the board directory when quarantine is unusable. + if board_descriptor is not None and source_stat is not None: try: - os.unlink(path.name, dir_fd=board_descriptor) - retired = True - os.fsync(board_descriptor) + retirement = _retire_terminal_entry( + path, + source_stat, + board_descriptor=board_descriptor, + quarantine_descriptor=( + quarantine_descriptor + if quarantine_descriptor is not None + else board_descriptor + ), + ) except OSError: - pass + retirement = "deferred" print( f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}", file=sys.stderr, flush=True, ) finally: + if source_descriptor is not None: + os.close(source_descriptor) if quarantine_descriptor is not None: os.close(quarantine_descriptor) if board_descriptor is not None: os.close(board_descriptor) print( "quarantined terminal journal " - f"{path.name}: {reason}; identity={identity or 'unparseable'}", + f"{path.name}: {reason}; identity={identity or 'unparseable'}; " + f"retirement={retirement}", file=sys.stderr, flush=True, ) @@ -1356,6 +1498,23 @@ def _finalize_terminal_record( return outcome +def _replay_replacement_terminal( + kanban_db: Any, + path: Path, + identity: TerminalIdentity, +) -> tuple[bool, bool]: + """Replay a valid journal atomically substituted during quarantine.""" + replacement = _load_terminal_json(path, identity) + if not _terminal_record_valid(replacement, identity): + return False, False + try: + outcome = _finalize_terminal_record(kanban_db, path, replacement) + except Exception as error: + _record_board_access_error(identity.board, error) + return True, False + return True, outcome == "committed" + + def recover_pending_finalizations() -> int: """Replay accepted exact-run results before scheduling more provider work.""" from hermes_cli import kanban_db @@ -1369,11 +1528,23 @@ def recover_pending_finalizations() -> int: continue if not _terminal_record_valid(record): _quarantine_terminal(path, identity, "malformed-payload") - _recover_exact_run(kanban_db, identity, "malformed-payload") + replacement, committed = _replay_replacement_terminal( + kanban_db, path, identity + ) + if committed: + recovered += 1 + elif not replacement: + _recover_exact_run(kanban_db, identity, "malformed-payload") continue if not _terminal_record_valid(record, identity): _quarantine_terminal(path, identity, "foreign-identity") - _recover_exact_run(kanban_db, identity, "foreign-identity") + replacement, committed = _replay_replacement_terminal( + kanban_db, path, identity + ) + if committed: + recovered += 1 + elif not replacement: + _recover_exact_run(kanban_db, identity, "foreign-identity") continue try: outcome = _finalize_terminal_record(kanban_db, path, record) @@ -1384,7 +1555,13 @@ def recover_pending_finalizations() -> int: recovered += 1 elif outcome in {"invalid", "foreign", "stale"}: _quarantine_terminal(path, identity, outcome) - _recover_exact_run(kanban_db, identity, outcome) + replacement, committed = _replay_replacement_terminal( + kanban_db, path, identity + ) + if committed: + recovered += 1 + elif not replacement: + _recover_exact_run(kanban_db, identity, outcome) return recovered @@ -1422,6 +1599,27 @@ def _artifact_gc_candidates(board_dir: Path) -> list[Path]: return candidates +def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool: + """Remove only the exact artifact inode selected by retention.""" + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = None + try: + descriptor = os.open(path.parent, directory_flags) + outcome = _retire_terminal_entry( + path, + observed, + board_descriptor=descriptor, + quarantine_descriptor=descriptor, + ) + return outcome == "retired" + except OSError: + return False + finally: + if descriptor is not None: + os.close(descriptor) + + def gc_lane_artifacts( *, now: float | None = None, @@ -1443,14 +1641,14 @@ def gc_lane_artifacts( except OSError: continue if not stat.S_ISREG(file_stat.st_mode): - path.unlink() - removed += 1 - board_removed += 1 + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 continue if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds: - path.unlink() - removed += 1 - board_removed += 1 + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 else: entries.append((path, file_stat)) retained_count = 0 @@ -1463,9 +1661,9 @@ def gc_lane_artifacts( max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes ) if exceeds_count or exceeds_bytes: - path.unlink() - removed += 1 - board_removed += 1 + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 continue retained_count += 1 retained_bytes += file_stat.st_size diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 2db9b1ae..94d9cb58 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -8,6 +8,7 @@ import hashlib import json import os import signal +import stat import sys from contextlib import nullcontext from pathlib import Path @@ -1300,8 +1301,11 @@ def test_quarantine_avoids_symlink_and_mode_collision_destinations( assert not path.exists() assert destination.name == f"{base}.2.quarantine" - assert destination.read_bytes() == b"invalid" assert destination.stat().st_mode & 0o777 == 0o600 + diagnostic = json.loads(destination.read_text(encoding="utf-8")) + assert diagnostic["size"] == len(b"invalid") + assert diagnostic["sha256"] == hashlib.sha256(b"invalid").hexdigest() + assert diagnostic["source_kind"] == "regular" assert victim.read_text(encoding="utf-8") == "unchanged" assert collision.read_text(encoding="utf-8") == "attacker collision" assert collision.stat().st_mode & 0o777 == 0o644 @@ -1339,6 +1343,241 @@ def test_hardlinked_terminal_source_is_never_chmodded_or_copied_as_data( assert "foreign inode contents" not in destination.read_text(encoding="utf-8") +def test_quarantine_preserves_and_replays_an_atomic_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_swap"), 21) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed") + identity = lanes._terminal_identity(path) + assert identity is not None + structured = _completed_result("replacement result") + replacement_record = { + "board": "cassandra", + "task_id": "t_swap", + "expected_run_id": 21, + "result": json.dumps(structured, sort_keys=True), + "summary": structured["summary"], + "metadata": {}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + replacement = path.with_name("replacement.tmp") + replacement.write_text( + json.dumps(replacement_record, sort_keys=True), + encoding="utf-8", + ) + replacement.chmod(0o600) + real_fsync = lanes.os.fsync + swapped = {"value": False} + + def swap_on_quarantine_directory_fsync(descriptor): + descriptor_stat = os.fstat(descriptor) + if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + real_fsync(descriptor) + + monkeypatch.setattr(lanes.os, "fsync", swap_on_quarantine_directory_fsync) + + lanes._quarantine_terminal(path, identity, "malformed-payload") + + assert swapped["value"] is True + assert path.exists() + assert lanes._terminal_record_valid( + lanes._load_terminal_json(path, identity), identity + ) + + task = SimpleNamespace( + id="t_swap", status="running", current_run_id=21, result=None, assignee="cli-auto" + ) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert not path.exists() + + +def test_invalid_utf8_journal_quarantines_and_does_not_stop_later_replay( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + bad = lanes._terminal_path(lanes.state_path("cassandra", "a_bad_utf8"), 31) + bad.parent.mkdir(parents=True) + bad.write_bytes(b"\xff\xfe\x80not-json") + good, _record = lanes._write_terminal_record( + lanes.state_path("cassandra", "z_good"), + board="cassandra", + task_id="z_good", + run_id=32, + structured=_completed_result("valid after invalid UTF-8"), + summary="valid after invalid UTF-8", + metadata={}, + ) + tasks = { + "a_bad_utf8": SimpleNamespace( + id="a_bad_utf8", + status="running", + current_run_id=31, + result=None, + assignee="cli-auto", + ), + "z_good": SimpleNamespace( + id="z_good", + status="running", + current_run_id=32, + result=None, + assignee="cli-auto", + ), + } + + class Connection: + def close(self): + return None + + def reclaim_task(_conn, task_id, **_kwargs): + tasks[task_id].status = "ready" + tasks[task_id].current_run_id = None + return True + + def complete_task(_conn, task_id, **kwargs): + tasks[task_id].status = "done" + tasks[task_id].result = kwargs["result"] + tasks[task_id].current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, task_id: tasks[task_id], + reclaim_task=reclaim_task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert not bad.exists() + assert not good.exists() + assert tasks["a_bad_utf8"].status == "ready" + assert tasks["z_good"].status == "done" + diagnostic_path = next((bad.parent / "quarantine").glob("*.quarantine")) + diagnostic = json.loads(diagnostic_path.read_text(encoding="utf-8")) + assert diagnostic["size"] == len(b"\xff\xfe\x80not-json") + assert diagnostic_path.stat().st_size < 4096 + + +@pytest.mark.parametrize( + "size", + [lanes.MAX_TERMINAL_RECORD_BYTES + 1, 16 * 1024 * 1024], +) +def test_oversized_sparse_journal_has_bounded_read_and_quarantine( + tmp_path: Path, + monkeypatch, + size: int, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + huge = lanes._terminal_path(lanes.state_path("cassandra", "a_huge"), 41) + huge.parent.mkdir(parents=True) + with huge.open("wb") as stream: + stream.seek(size - 1) + stream.write(b"\0") + good, _record = lanes._write_terminal_record( + lanes.state_path("cassandra", "z_after_huge"), + board="cassandra", + task_id="z_after_huge", + run_id=42, + structured=_completed_result("valid after oversized journal"), + summary="valid after oversized journal", + metadata={}, + ) + tasks = { + "a_huge": SimpleNamespace( + id="a_huge", + status="running", + current_run_id=41, + result=None, + assignee="cli-auto", + ), + "z_after_huge": SimpleNamespace( + id="z_after_huge", + status="running", + current_run_id=42, + result=None, + assignee="cli-auto", + ), + } + reads = [] + real_read_bounded = lanes._read_bounded + + def observe_read_limit(descriptor, limit): + reads.append((os.fstat(descriptor).st_size, limit)) + return real_read_bounded(descriptor, limit) + + monkeypatch.setattr(lanes, "_read_bounded", observe_read_limit) + + class Connection: + def close(self): + return None + + def reclaim_task(_conn, task_id, **_kwargs): + tasks[task_id].status = "ready" + tasks[task_id].current_run_id = None + return True + + def complete_task(_conn, task_id, **kwargs): + tasks[task_id].status = "done" + tasks[task_id].result = kwargs["result"] + tasks[task_id].current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, task_id: tasks[task_id], + reclaim_task=reclaim_task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert not huge.exists() + assert not good.exists() + assert tasks["a_huge"].status == "ready" + assert tasks["z_after_huge"].status == "done" + diagnostics = list((huge.parent / "quarantine").glob("*.quarantine")) + assert len(diagnostics) == 1 + assert diagnostics[0].stat().st_size < 4096 + diagnostic = json.loads(diagnostics[0].read_text(encoding="utf-8")) + assert diagnostic["size"] == size + assert diagnostic["hashed_bytes"] == lanes.QUARANTINE_HASH_BYTES + assert diagnostic["hash_complete"] is False + assert (size, lanes.QUARANTINE_HASH_BYTES) in reads + assert (size, lanes.MAX_TERMINAL_RECORD_BYTES + 1) not in reads + assert max(limit for _file_size, limit in reads) <= ( + lanes.MAX_TERMINAL_RECORD_BYTES + 1 + ) + + def test_recovery_does_not_parse_committed_journals_every_tick( tmp_path: Path, monkeypatch, @@ -1453,6 +1692,36 @@ def test_artifact_gc_prunes_oldest_by_count_and_total_bytes( assert [path.exists() for path in files] == [False, False, True] +def test_artifact_gc_preserves_a_concurrent_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + path = board / "t.run-1.provider-1.result.json" + path.write_text("old", encoding="utf-8") + os.utime(path, (100, 100)) + replacement = board / "replacement.tmp" + replacement.write_text("new", encoding="utf-8") + real_unlink = lanes._unlink_artifact_if_same + swapped = {"value": False} + + def swap_before_identity_checked(candidate, observed): + if not swapped["value"]: + os.replace(replacement, candidate) + swapped["value"] = True + return real_unlink(candidate, observed) + + monkeypatch.setattr(lanes, "_unlink_artifact_if_same", swap_before_identity_checked) + + assert lanes.gc_lane_artifacts( + now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 + ) == 0 + assert swapped["value"] is True + assert path.read_text(encoding="utf-8") == "new" + + def test_artifact_gc_is_interval_gated( tmp_path: Path, monkeypatch, From 1ba9f139594e2b24fd9080a13c34b2198124faae Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 21:55:28 -0300 Subject: [PATCH 07/46] hermes: bind terminal commit to journal inode --- dockerfiles/Dockerfile.hermes-agent | 19 +- .../Dockerfile.hermes-agent.dockerignore | 10 + .../hermes-execution-safety-regression.py | 304 +- .../hermes_decomposition_safety_regression.py | 120 + dockerfiles/hermes_execution_patch_support.py | 12 + .../hermes_execution_regression_support.py | 132 + dockerfiles/hermes_run_safety_regression.py | 467 +++ dockerfiles/patch-hermes-execution-safety.py | 381 +- .../patch_hermes_decomposition_safety.py | 262 ++ dockerfiles/patch_hermes_run_safety.py | 480 +++ .../tests/test_dashboard_render_support.py | 25 + ...tlas.py => test_dashboards_render_jobs.py} | 246 +- .../tests/test_dashboards_render_overview.py | 221 ++ services/hermes/kustomization.yaml | 14 + services/hermes/scripts/cli_lane_board.py | 82 + services/hermes/scripts/cli_lane_config.py | 196 + services/hermes/scripts/cli_lane_dispatch.py | 159 + services/hermes/scripts/cli_lane_evidence.py | 286 ++ services/hermes/scripts/cli_lane_execution.py | 425 +++ services/hermes/scripts/cli_lane_files.py | 174 + .../hermes/scripts/cli_lane_finalization.py | 323 ++ services/hermes/scripts/cli_lane_prompt.py | 117 + services/hermes/scripts/cli_lane_provider.py | 406 +++ .../hermes/scripts/cli_lane_quarantine.py | 222 ++ services/hermes/scripts/cli_lane_records.py | 356 ++ services/hermes/scripts/cli_lane_recovery.py | 460 +++ services/hermes/scripts/cli_lane_retention.py | 193 + services/hermes/scripts/cli_lane_routing.py | 161 + services/hermes/scripts/cli_lane_runner.py | 2340 +----------- .../scripts/recover_cassandra_workers.py | 4 +- testing/tests/test_hermes_agent_access.py | 253 ++ testing/tests/test_hermes_agent_layout.py | 323 ++ .../test_hermes_agent_runtime_patches.py | 329 ++ testing/tests/test_hermes_agent_security.py | 202 + .../tests/test_hermes_chat_codex_broker.py | 75 + testing/tests/test_hermes_chat_config.py | 273 ++ testing/tests/test_hermes_chat_images.py | 284 ++ .../tests/test_hermes_chat_local_compute.py | 295 ++ .../tests/test_hermes_chat_provider_auth.py | 192 + testing/tests/test_hermes_chat_quality.py | 2242 ------------ testing/tests/test_hermes_chat_sandbox.py | 24 + testing/tests/test_hermes_chat_sessions.py | 392 ++ testing/tests/test_hermes_chat_support.py | 73 + testing/tests/test_hermes_chat_switchyard.py | 483 +++ testing/tests/test_hermes_chat_voice.py | 265 ++ testing/tests/test_hermes_cli_dispatch.py | 420 +++ testing/tests/test_hermes_cli_durability.py | 361 ++ testing/tests/test_hermes_cli_fallback.py | 381 ++ testing/tests/test_hermes_cli_finalization.py | 411 +++ .../test_hermes_cli_finalization_authority.py | 53 + .../tests/test_hermes_cli_goal_execution.py | 151 + testing/tests/test_hermes_cli_lane_routing.py | 2 +- testing/tests/test_hermes_cli_lanes.py | 3239 ----------------- testing/tests/test_hermes_cli_provenance.py | 313 ++ .../test_hermes_cli_provider_sessions.py | 301 ++ testing/tests/test_hermes_cli_quarantine.py | 375 ++ testing/tests/test_hermes_cli_retention.py | 325 ++ testing/tests/test_hermes_cli_routing.py | 155 + testing/tests/test_hermes_cli_run_identity.py | 217 ++ testing/tests/test_hermes_cli_staging.py | 441 +++ testing/tests/test_hermes_cli_support.py | 236 ++ .../tests/test_hermes_coordinator_boards.py | 204 ++ ....py => test_hermes_coordinator_routing.py} | 256 +- .../tests/test_hermes_coordinator_support.py | 70 + testing/tests/test_hermes_worker_recovery.py | 22 +- 65 files changed, 13368 insertions(+), 8867 deletions(-) create mode 100644 dockerfiles/hermes_decomposition_safety_regression.py create mode 100644 dockerfiles/hermes_execution_patch_support.py create mode 100644 dockerfiles/hermes_execution_regression_support.py create mode 100644 dockerfiles/hermes_run_safety_regression.py create mode 100644 dockerfiles/patch_hermes_decomposition_safety.py create mode 100644 dockerfiles/patch_hermes_run_safety.py create mode 100644 scripts/tests/test_dashboard_render_support.py rename scripts/tests/{test_dashboards_render_atlas.py => test_dashboards_render_jobs.py} (54%) create mode 100644 scripts/tests/test_dashboards_render_overview.py create mode 100644 services/hermes/scripts/cli_lane_board.py create mode 100644 services/hermes/scripts/cli_lane_config.py create mode 100644 services/hermes/scripts/cli_lane_dispatch.py create mode 100644 services/hermes/scripts/cli_lane_evidence.py create mode 100644 services/hermes/scripts/cli_lane_execution.py create mode 100644 services/hermes/scripts/cli_lane_files.py create mode 100644 services/hermes/scripts/cli_lane_finalization.py create mode 100644 services/hermes/scripts/cli_lane_prompt.py create mode 100644 services/hermes/scripts/cli_lane_provider.py create mode 100644 services/hermes/scripts/cli_lane_quarantine.py create mode 100644 services/hermes/scripts/cli_lane_records.py create mode 100644 services/hermes/scripts/cli_lane_recovery.py create mode 100644 services/hermes/scripts/cli_lane_retention.py create mode 100644 services/hermes/scripts/cli_lane_routing.py create mode 100644 testing/tests/test_hermes_agent_access.py create mode 100644 testing/tests/test_hermes_agent_layout.py create mode 100644 testing/tests/test_hermes_agent_runtime_patches.py create mode 100644 testing/tests/test_hermes_agent_security.py create mode 100644 testing/tests/test_hermes_chat_codex_broker.py create mode 100644 testing/tests/test_hermes_chat_config.py create mode 100644 testing/tests/test_hermes_chat_images.py create mode 100644 testing/tests/test_hermes_chat_local_compute.py create mode 100644 testing/tests/test_hermes_chat_provider_auth.py delete mode 100644 testing/tests/test_hermes_chat_quality.py create mode 100644 testing/tests/test_hermes_chat_sandbox.py create mode 100644 testing/tests/test_hermes_chat_sessions.py create mode 100644 testing/tests/test_hermes_chat_support.py create mode 100644 testing/tests/test_hermes_chat_switchyard.py create mode 100644 testing/tests/test_hermes_chat_voice.py create mode 100644 testing/tests/test_hermes_cli_dispatch.py create mode 100644 testing/tests/test_hermes_cli_durability.py create mode 100644 testing/tests/test_hermes_cli_fallback.py create mode 100644 testing/tests/test_hermes_cli_finalization.py create mode 100644 testing/tests/test_hermes_cli_finalization_authority.py create mode 100644 testing/tests/test_hermes_cli_goal_execution.py delete mode 100644 testing/tests/test_hermes_cli_lanes.py create mode 100644 testing/tests/test_hermes_cli_provenance.py create mode 100644 testing/tests/test_hermes_cli_provider_sessions.py create mode 100644 testing/tests/test_hermes_cli_quarantine.py create mode 100644 testing/tests/test_hermes_cli_retention.py create mode 100644 testing/tests/test_hermes_cli_routing.py create mode 100644 testing/tests/test_hermes_cli_run_identity.py create mode 100644 testing/tests/test_hermes_cli_staging.py create mode 100644 testing/tests/test_hermes_cli_support.py create mode 100644 testing/tests/test_hermes_coordinator_boards.py rename testing/tests/{test_hermes_coordinator.py => test_hermes_coordinator_routing.py} (54%) create mode 100644 testing/tests/test_hermes_coordinator_support.py diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index fc99307e..e0024e44 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -1546,10 +1546,25 @@ NODE COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py +COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py +COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py +COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py +COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py +COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py +COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py +COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/ RUN /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \ - && /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \ + && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \ + /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \ && rm /tmp/patch-hermes-execution-safety.py \ - /tmp/hermes-execution-safety-regression.py + /tmp/hermes-execution-safety-regression.py \ + /tmp/hermes_execution_patch_support.py \ + /tmp/patch_hermes_run_safety.py \ + /tmp/patch_hermes_decomposition_safety.py \ + /tmp/hermes_execution_regression_support.py \ + /tmp/hermes_run_safety_regression.py \ + /tmp/hermes_decomposition_safety_regression.py \ + && find /tmp/hermes-lane-regression -depth -delete COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index 1b3f891a..e6cb3493 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -7,3 +7,13 @@ !dockerfiles/hermes-session-migrate.py !dockerfiles/patch-hermes-execution-safety.py !dockerfiles/hermes-execution-safety-regression.py +!dockerfiles/hermes_execution_patch_support.py +!dockerfiles/patch_hermes_run_safety.py +!dockerfiles/patch_hermes_decomposition_safety.py +!dockerfiles/hermes_execution_regression_support.py +!dockerfiles/hermes_run_safety_regression.py +!dockerfiles/hermes_decomposition_safety_regression.py +!services/ +!services/hermes/ +!services/hermes/scripts/ +!services/hermes/scripts/cli_lane_*.py diff --git a/dockerfiles/hermes-execution-safety-regression.py b/dockerfiles/hermes-execution-safety-regression.py index a77ca92d..2892c4c3 100644 --- a/dockerfiles/hermes-execution-safety-regression.py +++ b/dockerfiles/hermes-execution-safety-regression.py @@ -1,297 +1,23 @@ -"""Build-time regressions for Hermes automatic decomposition safety.""" +"""Run the composable Hermes execution-safety build regressions.""" -from __future__ import annotations - -import json -import os -import tempfile import unittest -from types import SimpleNamespace -from unittest import mock + +from hermes_decomposition_safety_regression import DecompositionSafetyTests +from hermes_run_safety_regression import ExactRunSafetyTests -_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-") -os.environ["HERMES_HOME"] = _HERMES_HOME.name - -from agent import auxiliary_client # noqa: E402 -from hermes_cli import kanban_db # noqa: E402 -from hermes_cli import kanban_decompose # noqa: E402 - - -class _FakeCompletions: - def __init__(self, payload: dict, before_response=None) -> None: - self.payload = payload - self.before_response = before_response - self.calls = 0 - - def create(self, **_kwargs): - self.calls += 1 - if self.before_response is not None: - self.before_response() - return SimpleNamespace( - choices=[ - SimpleNamespace( - message=SimpleNamespace(content=json.dumps(self.payload)) - ) - ] +def main() -> int: + """Run both transaction domains and return a build-friendly status.""" + loader = unittest.defaultTestLoader + suite = unittest.TestSuite( + ( + loader.loadTestsFromTestCase(ExactRunSafetyTests), + loader.loadTestsFromTestCase(DecompositionSafetyTests), ) - - -class _FakeClient: - def __init__(self, completions: _FakeCompletions) -> None: - self.chat = SimpleNamespace(completions=completions) - - -class AutomaticDecompositionSafetyTests(unittest.TestCase): - """Exercise the patched upstream APIs against an isolated real database.""" - - def setUp(self) -> None: - self.connection = kanban_db.connect() - self.patches = [ - mock.patch.object( - kanban_decompose, - "_build_roster", - return_value=([], {"default"}), - ), - mock.patch.object( - kanban_decompose, - "_load_config", - return_value={"kanban": {"auto_promote_children": True}}, - ), - mock.patch.object( - auxiliary_client, - "get_auxiliary_extra_body", - return_value={}, - ), - ] - for patch in self.patches: - patch.start() - - def tearDown(self) -> None: - for patch in reversed(self.patches): - patch.stop() - self.connection.close() - - def _client(self, payload: dict, before_response=None) -> _FakeCompletions: - completions = _FakeCompletions(payload, before_response) - client = _FakeClient(completions) - patch = mock.patch.object( - auxiliary_client, - "get_text_auxiliary_client", - return_value=(client, "test-decomposer"), - ) - patch.start() - self.addCleanup(patch.stop) - return completions - - def _status(self, task_id: str) -> str: - task = kanban_db.get_task(self.connection, task_id) - self.assertIsNotNone(task) - return task.status - - def _route_executed_task_to_triage(self) -> str: - task_id = kanban_db.create_task( - self.connection, - title="already implemented parent", - ) - for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT): - if attempt: - self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) - self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) - self.assertTrue( - kanban_db.block_task( - self.connection, - task_id, - reason="same unavailable capability", - kind="capability", - ) - ) - self.assertEqual(self._status(task_id), "triage") - self.assertEqual( - len(kanban_db.list_runs(self.connection, task_id)), - kanban_db.BLOCK_RECURRENCE_LIMIT, - ) - return task_id - - def test_automatic_decomposer_skips_executed_triage_task(self) -> None: - task_id = self._route_executed_task_to_triage() - completions = self._client( - {"fanout": False, "title": "must not run", "body": "must not run"} - ) - before = len(kanban_db.list_tasks(self.connection)) - - outcome = kanban_decompose.decompose_task(task_id, automatic=True) - - self.assertFalse(outcome.ok) - self.assertIn("execution history", outcome.reason) - self.assertEqual(completions.calls, 0) - self.assertEqual(self._status(task_id), "triage") - self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) - - def test_manual_decomposition_remains_available_after_execution(self) -> None: - task_id = self._route_executed_task_to_triage() - completions = self._client( - { - "fanout": False, - "title": "operator-approved retry", - "body": "Retain one bounded task.", - "assignee": "default", - } - ) - - outcome = kanban_decompose.decompose_task(task_id, author="operator") - - self.assertTrue(outcome.ok) - self.assertFalse(outcome.fanout) - self.assertEqual(completions.calls, 1) - self.assertEqual(self._status(task_id), "ready") - - def test_latest_ended_run_can_replay_a_durable_completion(self) -> None: - task_id = kanban_db.create_task(self.connection, title="journaled result") - self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) - run_id = kanban_db.get_task(self.connection, task_id).current_run_id - self.assertTrue( - kanban_db.block_task( - self.connection, - task_id, - reason="legacy post-journal failure", - kind="capability", - expected_run_id=run_id, - ) - ) - - self.assertTrue( - kanban_db.complete_task( - self.connection, - task_id, - result="durable terminal result", - summary="durable terminal result", - replay_ended_run_id=run_id, - ) - ) - self.assertEqual(self._status(task_id), "done") - - def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None: - task_id = kanban_db.create_task(self.connection, title="replacement guard") - self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) - old_run = kanban_db.get_task(self.connection, task_id).current_run_id - self.assertTrue( - kanban_db.block_task( - self.connection, - task_id, - reason="first run", - kind="capability", - expected_run_id=old_run, - ) - ) - self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) - self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) - replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id - self.assertNotEqual(old_run, replacement_run) - self.assertTrue( - kanban_db.block_task( - self.connection, - task_id, - reason="replacement run", - kind="capability", - expected_run_id=replacement_run, - ) - ) - - self.assertFalse( - kanban_db.complete_task( - self.connection, - task_id, - result="stale result", - replay_ended_run_id=old_run, - ) - ) - self.assertIn(self._status(task_id), {"blocked", "triage"}) - - def test_fresh_triage_task_still_auto_promotes(self) -> None: - task_id = kanban_db.create_task( - self.connection, - title="fresh objective", - triage=True, - ) - completions = self._client( - { - "fanout": False, - "title": "specified objective", - "body": "One bounded task.", - "assignee": "default", - } - ) - - outcome = kanban_decompose.decompose_task(task_id, automatic=True) - - self.assertTrue(outcome.ok) - self.assertEqual(completions.calls, 1) - self.assertEqual(self._status(task_id), "ready") - - def _insert_run(self, task_id: str) -> None: - with kanban_db.connect_closing() as connection: - kanban_db._synthesize_ended_run( - connection, - task_id, - outcome="reclaimed", - summary="concurrent execution evidence", - ) - - def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None: - task_id = kanban_db.create_task( - self.connection, - title="concurrent single objective", - triage=True, - ) - completions = self._client( - { - "fanout": False, - "title": "redundant rewrite", - "body": "must not be committed", - "assignee": "default", - }, - before_response=lambda: self._insert_run(task_id), - ) - - outcome = kanban_decompose.decompose_task(task_id, automatic=True) - - self.assertFalse(outcome.ok) - self.assertIn("gained execution history", outcome.reason) - self.assertEqual(completions.calls, 1) - self.assertEqual(self._status(task_id), "triage") - - def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None: - task_id = kanban_db.create_task( - self.connection, - title="concurrent triage objective", - triage=True, - ) - - completions = self._client( - { - "fanout": True, - "tasks": [ - { - "title": "redundant child", - "body": "must not be created", - "assignee": "default", - "parents": [], - } - ], - }, - before_response=lambda: self._insert_run(task_id), - ) - before = len(kanban_db.list_tasks(self.connection)) - - outcome = kanban_decompose.decompose_task(task_id, automatic=True) - - self.assertFalse(outcome.ok) - self.assertIn("gained execution history", outcome.reason) - self.assertEqual(completions.calls, 1) - self.assertEqual(self._status(task_id), "triage") - self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) + ) + result = unittest.TextTestRunner(verbosity=2).run(suite) + return 0 if result.wasSuccessful() else 1 if __name__ == "__main__": - unittest.main(verbosity=2) + raise SystemExit(main()) diff --git a/dockerfiles/hermes_decomposition_safety_regression.py b/dockerfiles/hermes_decomposition_safety_regression.py new file mode 100644 index 00000000..d1eafe23 --- /dev/null +++ b/dockerfiles/hermes_decomposition_safety_regression.py @@ -0,0 +1,120 @@ +"""Regress automatic decomposition against existing execution history.""" + +from __future__ import annotations + + +from hermes_cli import kanban_db +from hermes_cli import kanban_decompose +from hermes_execution_regression_support import ExecutionSafetyTestCase + + +class DecompositionSafetyTests(ExecutionSafetyTestCase): + """Verify only unexecuted triage roots may decompose automatically.""" + + def test_automatic_decomposer_skips_executed_triage_task(self) -> None: + task_id = self._route_executed_task_to_triage() + completions = self._client( + {"fanout": False, "title": "must not run", "body": "must not run"} + ) + before = len(kanban_db.list_tasks(self.connection)) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("execution history", outcome.reason) + self.assertEqual(completions.calls, 0) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) + + def test_manual_decomposition_remains_available_after_execution(self) -> None: + task_id = self._route_executed_task_to_triage() + completions = self._client( + { + "fanout": False, + "title": "operator-approved retry", + "body": "Retain one bounded task.", + "assignee": "default", + } + ) + + outcome = kanban_decompose.decompose_task(task_id, author="operator") + + self.assertTrue(outcome.ok) + self.assertFalse(outcome.fanout) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "ready") + + def test_fresh_triage_task_still_auto_promotes(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="fresh objective", + triage=True, + ) + completions = self._client( + { + "fanout": False, + "title": "specified objective", + "body": "One bounded task.", + "assignee": "default", + } + ) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertTrue(outcome.ok) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "ready") + + def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="concurrent single objective", + triage=True, + ) + completions = self._client( + { + "fanout": False, + "title": "redundant rewrite", + "body": "must not be committed", + "assignee": "default", + }, + before_response=lambda: self._insert_run(task_id), + ) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("gained execution history", outcome.reason) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "triage") + + def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None: + task_id = kanban_db.create_task( + self.connection, + title="concurrent triage objective", + triage=True, + ) + + completions = self._client( + { + "fanout": True, + "tasks": [ + { + "title": "redundant child", + "body": "must not be created", + "assignee": "default", + "parents": [], + } + ], + }, + before_response=lambda: self._insert_run(task_id), + ) + before = len(kanban_db.list_tasks(self.connection)) + + outcome = kanban_decompose.decompose_task(task_id, automatic=True) + + self.assertFalse(outcome.ok) + self.assertIn("gained execution history", outcome.reason) + self.assertEqual(completions.calls, 1) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) diff --git a/dockerfiles/hermes_execution_patch_support.py b/dockerfiles/hermes_execution_patch_support.py new file mode 100644 index 00000000..32ab8e2c --- /dev/null +++ b/dockerfiles/hermes_execution_patch_support.py @@ -0,0 +1,12 @@ +"""Fail-closed helpers shared by Hermes source patches.""" + + + +def replace_once(source: str, before: str, after: str, label: str) -> str: + """Replace one anchored upstream fragment and fail closed on source drift.""" + count = source.count(before) + if count != 1: + raise SystemExit( + f"Hermes {label} patch context changed: expected 1, found {count}" + ) + return source.replace(before, after, 1) diff --git a/dockerfiles/hermes_execution_regression_support.py b/dockerfiles/hermes_execution_regression_support.py new file mode 100644 index 00000000..aa7d576a --- /dev/null +++ b/dockerfiles/hermes_execution_regression_support.py @@ -0,0 +1,132 @@ +"""Shared isolated database fixture for Hermes execution-safety regressions.""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-") +os.environ["HERMES_HOME"] = _HERMES_HOME.name +_LANE_SOURCE = Path( + os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression") +) +sys.path.insert(0, str(_LANE_SOURCE)) + +from agent import auxiliary_client # noqa: E402 +from cli_lane_config import TerminalIdentity # noqa: E402 +from cli_lane_finalization import _finalize_document_db # noqa: E402 +from hermes_cli import kanban_db # noqa: E402 +from hermes_cli import kanban_decompose # noqa: E402 + +__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db") + + +class _FakeCompletions: + def __init__(self, payload: dict, before_response=None) -> None: + self.payload = payload + self.before_response = before_response + self.calls = 0 + + def create(self, **_kwargs): + self.calls += 1 + if self.before_response is not None: + self.before_response() + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content=json.dumps(self.payload)) + ) + ] + ) + + +class _FakeClient: + def __init__(self, completions: _FakeCompletions) -> None: + self.chat = SimpleNamespace(completions=completions) + + +class ExecutionSafetyTestCase(unittest.TestCase): + """Exercise patched upstream APIs against an isolated real database.""" + + def setUp(self) -> None: + self.connection = kanban_db.connect() + self.patches = [ + mock.patch.object( + kanban_decompose, + "_build_roster", + return_value=([], {"default"}), + ), + mock.patch.object( + kanban_decompose, + "_load_config", + return_value={"kanban": {"auto_promote_children": True}}, + ), + mock.patch.object( + auxiliary_client, + "get_auxiliary_extra_body", + return_value={}, + ), + ] + for patch in self.patches: + patch.start() + + def tearDown(self) -> None: + for patch in reversed(self.patches): + patch.stop() + self.connection.close() + + def _client(self, payload: dict, before_response=None) -> _FakeCompletions: + completions = _FakeCompletions(payload, before_response) + client = _FakeClient(completions) + patch = mock.patch.object( + auxiliary_client, + "get_text_auxiliary_client", + return_value=(client, "test-decomposer"), + ) + patch.start() + self.addCleanup(patch.stop) + return completions + + def _status(self, task_id: str) -> str: + task = kanban_db.get_task(self.connection, task_id) + self.assertIsNotNone(task) + return task.status + + def _route_executed_task_to_triage(self) -> str: + task_id = kanban_db.create_task( + self.connection, + title="already implemented parent", + ) + for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT): + if attempt: + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="same unavailable capability", + kind="capability", + ) + ) + self.assertEqual(self._status(task_id), "triage") + self.assertEqual( + len(kanban_db.list_runs(self.connection, task_id)), + kanban_db.BLOCK_RECURRENCE_LIMIT, + ) + return task_id + + def _insert_run(self, task_id: str) -> None: + with kanban_db.connect_closing() as connection: + kanban_db._synthesize_ended_run( + connection, + task_id, + outcome="reclaimed", + summary="concurrent execution evidence", + ) diff --git a/dockerfiles/hermes_run_safety_regression.py b/dockerfiles/hermes_run_safety_regression.py new file mode 100644 index 00000000..7f242e98 --- /dev/null +++ b/dockerfiles/hermes_run_safety_regression.py @@ -0,0 +1,467 @@ +"""Regress exact-run replay, provenance, and reclaim safety.""" + +from __future__ import annotations + +import threading +from unittest import mock + +from hermes_cli import kanban_db +from hermes_execution_regression_support import ( + ExecutionSafetyTestCase, + TerminalIdentity, + _finalize_document_db, +) +from cli_lane_files import state_path +from cli_lane_records import _write_terminal_record +from cli_lane_recovery import recover_pending_finalizations + + +class ExactRunSafetyTests(ExecutionSafetyTestCase): + """Verify exact-run transactions reject every stale concurrent actor.""" + + def _lane_document(self, summary: str) -> dict: + """Return the DB-bearing portion of one accepted lane journal.""" + return { + "result": summary, + "summary": summary, + "metadata": {"source": "exact-run-regression"}, + } + + def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None: + task_id = kanban_db.create_task(self.connection, title="replaced journal") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + first_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="first run ended after persistence", + expected_run_id=first_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + second_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="replacement run also ended", + expected_run_id=second_run, + ) + ) + before = kanban_db.get_task(self.connection, task_id) + + outcome = _finalize_document_db( + kanban_db, + TerminalIdentity("default", task_id, first_run, "pending"), + self._lane_document("stale first result"), + ) + + after = kanban_db.get_task(self.connection, task_id) + self.assertEqual(outcome, "stale") + self.assertEqual(after.status, before.status) + self.assertEqual(after.current_run_id, before.current_run_id) + self.assertEqual(after.result, before.result) + self.assertEqual(after.completed_run_id, before.completed_run_id) + + def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None: + task_id = kanban_db.create_task(self.connection, title="no such run") + before = kanban_db.get_task(self.connection, task_id) + + outcome = _finalize_document_db( + kanban_db, + TerminalIdentity("default", task_id, 42, "pending"), + self._lane_document("orphan result"), + ) + + after = kanban_db.get_task(self.connection, task_id) + self.assertEqual(outcome, "stale") + self.assertEqual(after.status, before.status) + self.assertEqual(after.result, before.result) + self.assertEqual(kanban_db.list_runs(self.connection, task_id), []) + + def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None: + task_id = kanban_db.create_task(self.connection, title="recovery conflict") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + first_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="first accepted run ended", + expected_run_id=first_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + second_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="replacement ended", + expected_run_id=second_run, + ) + ) + before = kanban_db.get_task(self.connection, task_id) + state_file = state_path("default", task_id) + pending, _record = _write_terminal_record( + state_file, + board="default", + task_id=task_id, + run_id=first_run, + structured={ + "status": "completed", + "summary": "accepted stale result", + "changed_files": [], + "tests_run": ["real SQLite recovery regression"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + summary="accepted stale result", + metadata={"source": "exact-run-regression"}, + ) + + self.assertEqual(recover_pending_finalizations(), 0) + + after = kanban_db.get_task(self.connection, task_id) + self.assertEqual(after.status, before.status) + self.assertEqual(after.current_run_id, before.current_run_id) + self.assertEqual(after.result, before.result) + self.assertFalse(pending.exists()) + conflicts = list( + state_file.parent.glob( + f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json" + ) + ) + self.assertEqual(len(conflicts), 1) + + def test_latest_ended_run_can_replay_a_durable_completion(self) -> None: + task_id = kanban_db.create_task(self.connection, title="journaled result") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="legacy post-journal failure", + kind="capability", + expected_run_id=run_id, + ) + ) + + with mock.patch.object( + kanban_db, "_fire_kanban_lifecycle_hook" + ) as lifecycle: + self.assertTrue( + kanban_db.complete_task( + self.connection, + task_id, + result="durable terminal result", + summary="durable terminal result", + replay_ended_run_id=run_id, + ) + ) + lifecycle.assert_called_once() + self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id)) + self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id) + task = kanban_db.get_task(self.connection, task_id) + self.assertEqual(task.status, "done") + self.assertEqual(task.completed_run_id, run_id) + runs = kanban_db.list_runs(self.connection, task_id) + self.assertEqual(len(runs), 1) + self.assertEqual(runs[0].id, run_id) + self.assertEqual(runs[0].status, "done") + self.assertEqual(runs[0].outcome, "completed") + self.assertEqual(runs[0].summary, "durable terminal result") + completed_events = self.connection.execute( + "SELECT run_id FROM task_events " + "WHERE task_id = ? AND kind = 'completed' ORDER BY id", + (task_id,), + ).fetchall() + self.assertEqual([row["run_id"] for row in completed_events], [run_id]) + + def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None: + task_id = self._route_executed_task_to_triage() + runs = kanban_db.list_runs(self.connection, task_id) + run_id = runs[-1].id + + self.assertTrue( + kanban_db.complete_task( + self.connection, + task_id, + result="accepted before the loop breaker routed triage", + summary="accepted before the loop breaker routed triage", + replay_ended_run_id=run_id, + ) + ) + + task = kanban_db.get_task(self.connection, task_id) + self.assertEqual(task.status, "done") + self.assertEqual(task.completed_run_id, run_id) + updated_runs = kanban_db.list_runs(self.connection, task_id) + self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT) + self.assertEqual(updated_runs[-1].id, run_id) + self.assertEqual(updated_runs[-1].outcome, "completed") + + def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None: + task_id = self._route_executed_task_to_triage() + run_id = kanban_db.list_runs(self.connection, task_id)[-1].id + self.assertTrue( + kanban_db.specify_triage_task( + self.connection, + task_id, + title="operator changed the objective", + author="operator", + ) + ) + + self.assertFalse( + kanban_db.complete_task( + self.connection, + task_id, + result="stale accepted result", + replay_ended_run_id=run_id, + ) + ) + self.assertNotEqual(self._status(task_id), "done") + + def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None: + task_id = self._route_executed_task_to_triage() + run_id = kanban_db.list_runs(self.connection, task_id)[-1].id + children = kanban_db.decompose_triage_task( + self.connection, + task_id, + root_assignee="default", + children=[{"title": "new child objective"}], + author="operator", + auto_promote=False, + ) + self.assertEqual(len(children or []), 1) + with kanban_db.write_txn(self.connection): + self.connection.execute( + "UPDATE tasks SET status = 'ready' WHERE id = ?", + (task_id,), + ) + + self.assertFalse( + kanban_db.complete_task( + self.connection, + task_id, + result="pre-decomposition accepted result", + replay_ended_run_id=run_id, + ) + ) + self.assertEqual(self._status(task_id), "ready") + + def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None: + task_id = kanban_db.create_task(self.connection, title="replacement guard") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + old_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="first run", + kind="capability", + expected_run_id=old_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertNotEqual(old_run, replacement_run) + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="replacement run", + kind="capability", + expected_run_id=replacement_run, + ) + ) + + self.assertFalse( + kanban_db.complete_task( + self.connection, + task_id, + result="stale result", + replay_ended_run_id=old_run, + ) + ) + self.assertIn(self._status(task_id), {"blocked", "triage"}) + + def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None: + task_id = kanban_db.create_task(self.connection, title="same result retries") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + first_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="first attempt ended", + expected_run_id=first_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + second_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.complete_task( + self.connection, + task_id, + result="identical result bytes", + summary="second run wins", + expected_run_id=second_run, + ) + ) + + task = kanban_db.get_task(self.connection, task_id) + self.assertEqual(task.completed_run_id, second_run) + self.assertNotEqual(task.completed_run_id, first_run) + runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)} + self.assertEqual(runs[first_run].outcome, "blocked") + self.assertEqual(runs[second_run].outcome, "completed") + + def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None: + task_id = kanban_db.create_task(self.connection, title="recover exact run") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + + self.assertTrue( + kanban_db.reclaim_task( + self.connection, + task_id, + reason="invalid exact journal", + expected_run_id=run_id, + ) + ) + task = kanban_db.get_task(self.connection, task_id) + self.assertEqual(task.status, "ready") + self.assertIsNone(task.current_run_id) + + def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None: + task_id = kanban_db.create_task(self.connection, title="replacement before txn") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + old_run = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="old recovery run", + expected_run_id=old_run, + ) + ) + self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + replacement = kanban_db.get_task(self.connection, task_id) + replacement_run = replacement.current_run_id + self.assertNotEqual(old_run, replacement_run) + signals = [] + + self.assertFalse( + kanban_db.reclaim_task( + self.connection, + task_id, + reason="stale journal", + expected_run_id=old_run, + signal_fn=lambda *args: signals.append(args), + ) + ) + latest = kanban_db.get_task(self.connection, task_id) + self.assertEqual(latest.status, "running") + self.assertEqual(latest.current_run_id, replacement_run) + self.assertEqual(latest.claim_lock, replacement.claim_lock) + self.assertEqual(signals, []) + + def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None: + task_id = kanban_db.create_task(self.connection, title="replacement in txn") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + old_run = kanban_db.get_task(self.connection, task_id).current_run_id + replacement = {} + + def install_replacement(*_args, **_kwargs): + cursor = self.connection.execute( + "INSERT INTO task_runs (task_id, status, claim_lock, started_at) " + "VALUES (?, 'running', ?, strftime('%s','now'))", + (task_id, "replacement-lock"), + ) + replacement["run_id"] = int(cursor.lastrowid) + self.connection.execute( + "UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?", + (replacement["run_id"], "replacement-lock", task_id), + ) + return {} + + with mock.patch.object( + kanban_db, + "_terminate_reclaimed_worker", + side_effect=install_replacement, + ): + self.assertFalse( + kanban_db.reclaim_task( + self.connection, + task_id, + reason="journal for old run", + expected_run_id=old_run, + ) + ) + + latest = kanban_db.get_task(self.connection, task_id) + self.assertEqual(latest.status, "running") + self.assertEqual(latest.current_run_id, replacement["run_id"]) + self.assertEqual(latest.claim_lock, "replacement-lock") + + def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None: + task_id = kanban_db.create_task(self.connection, title="concurrent finalizer") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + barrier = threading.Barrier(2) + outcomes = {} + errors = [] + + def reclaim() -> None: + try: + with kanban_db.connect_closing() as connection: + barrier.wait() + outcomes["reclaim"] = kanban_db.reclaim_task( + connection, + task_id, + reason="concurrent invalid journal", + expected_run_id=run_id, + ) + except BaseException as error: # pragma: no cover - assertion relay + errors.append(error) + + def finalize() -> None: + try: + with kanban_db.connect_closing() as connection: + barrier.wait() + outcomes["complete"] = kanban_db.complete_task( + connection, + task_id, + result="durable winner", + summary="durable winner", + expected_run_id=run_id, + ) + except BaseException as error: # pragma: no cover - assertion relay + errors.append(error) + + threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + self.assertFalse(thread.is_alive()) + + self.assertEqual(errors, []) + self.assertEqual(set(outcomes), {"reclaim", "complete"}) + self.assertEqual(sum(bool(value) for value in outcomes.values()), 1) + latest = kanban_db.get_task(self.connection, task_id) + self.assertIn(latest.status, {"done", "ready"}) + self.assertIsNone(latest.current_run_id) + runs = kanban_db.list_runs(self.connection, task_id) + self.assertEqual(len(runs), 1) + self.assertIsNotNone(runs[0].ended_at) diff --git a/dockerfiles/patch-hermes-execution-safety.py b/dockerfiles/patch-hermes-execution-safety.py index 6abd6f26..645dcb57 100644 --- a/dockerfiles/patch-hermes-execution-safety.py +++ b/dockerfiles/patch-hermes-execution-safety.py @@ -1,377 +1,6 @@ -"""Patch upstream Hermes exact-run completion and decomposition safety.""" +"""Apply the composable Hermes execution-safety source patches.""" -import os -from pathlib import Path - - -def replace_once(source: str, before: str, after: str, label: str) -> str: - """Replace one anchored upstream fragment and fail closed on source drift.""" - count = source.count(before) - if count != 1: - raise SystemExit( - f"Hermes {label} patch context changed: expected 1, found {count}" - ) - return source.replace(before, after, 1) - - -source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) -db_path = source_root / "hermes_cli/kanban_db.py" -db = db_path.read_text(encoding="utf-8") - -complete_signature_before = '''def complete_task( - conn: sqlite3.Connection, - task_id: str, - *, - result: Optional[str] = None, - summary: Optional[str] = None, - metadata: Optional[dict] = None, - created_cards: Optional[Iterable[str]] = None, - expected_run_id: Optional[int] = None, -) -> bool: -''' -complete_signature_after = '''def complete_task( - conn: sqlite3.Connection, - task_id: str, - *, - result: Optional[str] = None, - summary: Optional[str] = None, - metadata: Optional[dict] = None, - created_cards: Optional[Iterable[str]] = None, - expected_run_id: Optional[int] = None, - replay_ended_run_id: Optional[int] = None, -) -> bool: -''' -db = replace_once( - db, - complete_signature_before, - complete_signature_after, - "exact ended-run completion signature", -) - -complete_branch_before = ''' with write_txn(conn): - if expected_run_id is None: - cur = conn.execute( - """ - UPDATE tasks - SET status = 'done', - result = ?, - completed_at = ?, - claim_lock = NULL, - claim_expires= NULL, - worker_pid = NULL, - block_kind = NULL, - block_recurrences = 0 - WHERE id = ? - AND status IN ('running', 'ready', 'blocked', 'scheduled') - """, - (result, now, task_id), - ) - else: -''' -complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the - # accepted run. Verify that it is still the latest run in this same - # transaction; a replacement attempt must make the replay fail closed. - with write_txn(conn): - if expected_run_id is not None and replay_ended_run_id is not None: - raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive") - if replay_ended_run_id is not None: - cur = conn.execute( - """ - UPDATE tasks - SET status = 'done', - result = ?, - completed_at = ?, - claim_lock = NULL, - claim_expires= NULL, - worker_pid = NULL, - block_kind = NULL, - block_recurrences = 0 - WHERE id = ? - AND status IN ('ready', 'blocked') - AND current_run_id IS NULL - AND ? = ( - SELECT id FROM task_runs - WHERE task_id = ? - ORDER BY id DESC - LIMIT 1 - ) - """, - (result, now, task_id, int(replay_ended_run_id), task_id), - ) - elif expected_run_id is None: - cur = conn.execute( - """ - UPDATE tasks - SET status = 'done', - result = ?, - completed_at = ?, - claim_lock = NULL, - claim_expires= NULL, - worker_pid = NULL, - block_kind = NULL, - block_recurrences = 0 - WHERE id = ? - AND status IN ('running', 'ready', 'blocked', 'scheduled') - """, - (result, now, task_id), - ) - else: -''' -db = replace_once( - db, - complete_branch_before, - complete_branch_after, - "transactional exact ended-run completion", -) - -specify_signature_before = '''def specify_triage_task( - conn: sqlite3.Connection, - task_id: str, - *, - title: Optional[str] = None, - body: Optional[str] = None, - assignee: Optional[str] = None, - author: Optional[str] = None, -) -> bool: -''' -specify_signature_after = '''def specify_triage_task( - conn: sqlite3.Connection, - task_id: str, - *, - title: Optional[str] = None, - body: Optional[str] = None, - assignee: Optional[str] = None, - author: Optional[str] = None, - require_no_runs: bool = False, -) -> bool: -''' -db = replace_once( - db, - specify_signature_before, - specify_signature_after, - "transactional specify signature", -) - -specify_guard_before = ''' if existing is None: - return False - sets: list[str] = ["status = 'todo'"] -''' -specify_guard_after = ''' if existing is None: - return False - if require_no_runs and conn.execute( - "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", - (task_id,), - ).fetchone() is not None: - return False - sets: list[str] = ["status = 'todo'"] -''' -db = replace_once( - db, - specify_guard_before, - specify_guard_after, - "transactional specify execution-history guard", -) - -decompose_signature_before = '''def decompose_triage_task( - conn: sqlite3.Connection, - task_id: str, - *, - root_assignee: Optional[str], - children: list[dict], - author: Optional[str] = None, - auto_promote: bool = True, -) -> Optional[list[str]]: -''' -decompose_signature_after = '''def decompose_triage_task( - conn: sqlite3.Connection, - task_id: str, - *, - root_assignee: Optional[str], - children: list[dict], - author: Optional[str] = None, - auto_promote: bool = True, - require_no_runs: bool = False, -) -> Optional[list[str]]: -''' -db = replace_once( - db, - decompose_signature_before, - decompose_signature_after, - "transactional decomposition signature", -) - -decompose_guard_before = ''' if root_row["status"] != "triage": - return None - tenant = root_row["tenant"] -''' -decompose_guard_after = ''' if root_row["status"] != "triage": - return None - if require_no_runs and conn.execute( - "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", - (task_id,), - ).fetchone() is not None: - return None - tenant = root_row["tenant"] -''' -db = replace_once( - db, - decompose_guard_before, - decompose_guard_after, - "transactional decomposition execution-history guard", -) -db_path.write_text(db, encoding="utf-8") - -decompose_path = source_root / "hermes_cli/kanban_decompose.py" -decompose = decompose_path.read_text(encoding="utf-8") - -helper_anchor = ''' if chosen not in valid_names: - return default_assignee - return chosen - - -''' -helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool: - """Return whether a task has ever entered a worker run.""" - with kb.connect_closing() as conn: - return bool(kb.list_runs(conn, task_id)) - - -''' -decompose = replace_once( - decompose, - helper_anchor, - helper_replacement, - "auto-decompose execution-history helper", -) - -signature_before = '''def decompose_task( - task_id: str, - *, - author: Optional[str] = None, - timeout: Optional[int] = None, -) -> DecomposeOutcome: -''' -signature_after = '''def decompose_task( - task_id: str, - *, - author: Optional[str] = None, - timeout: Optional[int] = None, - automatic: bool = False, -) -> DecomposeOutcome: -''' -decompose = replace_once( - decompose, - signature_before, - signature_after, - "auto-decompose function signature", -) - -status_before = ''' if task.status != "triage": - return DecomposeOutcome( - task_id, False, f"task is not in triage (status={task.status!r})" - ) - - cfg = _load_config() -''' -status_after = ''' if task.status != "triage": - return DecomposeOutcome( - task_id, False, f"task is not in triage (status={task.status!r})" - ) - if automatic and _has_execution_history(task_id): - return DecomposeOutcome( - task_id, - False, - "task has execution history and requires deliberate manual triage", - ) - - cfg = _load_config() -''' -decompose = replace_once( - decompose, - status_before, - status_after, - "auto-decompose preflight guard", -) - -single_before = ''' author=audit_author, - ) - if not ok: -''' -single_after = ''' author=audit_author, - require_no_runs=automatic, - ) - if not ok: - if automatic and _has_execution_history(task_id): - return DecomposeOutcome( - task_id, - False, - "task gained execution history and requires deliberate manual triage", - ) -''' -decompose = replace_once( - decompose, - single_before, - single_after, - "auto-decompose single-task commit guard", -) - -fanout_before = ''' author=audit_author, - auto_promote=auto_promote, - ) -''' -fanout_after = ''' author=audit_author, - auto_promote=auto_promote, - require_no_runs=automatic, - ) -''' -decompose = replace_once( - decompose, - fanout_before, - fanout_after, - "auto-decompose transactional fanout guard", -) - -fanout_outcome_before = ''' if child_ids is None: - return DecomposeOutcome( - task_id, False, "task moved out of triage before decomposition", - ) -''' -fanout_outcome_after = ''' if child_ids is None: - if automatic and _has_execution_history(task_id): - return DecomposeOutcome( - task_id, - False, - "task gained execution history and requires deliberate manual triage", - ) - return DecomposeOutcome( - task_id, False, "task moved out of triage before decomposition", - ) -''' -decompose = replace_once( - decompose, - fanout_outcome_before, - fanout_outcome_after, - "auto-decompose fanout rejection reason", -) -decompose_path.write_text(decompose, encoding="utf-8") - - -watcher_path = source_root / "gateway/kanban_watchers.py" -watcher = watcher_path.read_text(encoding="utf-8") -call_before = ''' outcome = _decomp.decompose_task( - tid, author="auto-decomposer", - ) -''' -call_after = ''' outcome = _decomp.decompose_task( - tid, - author="auto-decomposer", - automatic=True, - ) -''' -watcher = replace_once( - watcher, - call_before, - call_after, - "gateway automatic-decomposition call", -) -watcher_path.write_text(watcher, encoding="utf-8") +# Import order is intentional: the decomposition DB contract builds on the +# exact-run schema and transaction changes. Each module fails closed on drift. +import patch_hermes_run_safety # noqa: F401 +import patch_hermes_decomposition_safety # noqa: F401 diff --git a/dockerfiles/patch_hermes_decomposition_safety.py b/dockerfiles/patch_hermes_decomposition_safety.py new file mode 100644 index 00000000..05c77356 --- /dev/null +++ b/dockerfiles/patch_hermes_decomposition_safety.py @@ -0,0 +1,262 @@ +"""Patch transactional automatic-decomposition execution-history guards.""" + +import os +from pathlib import Path + +from hermes_execution_patch_support import replace_once + +source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) +db_path = source_root / "hermes_cli/kanban_db.py" +db = db_path.read_text(encoding="utf-8") + +specify_signature_before = '''def specify_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + title: Optional[str] = None, + body: Optional[str] = None, + assignee: Optional[str] = None, + author: Optional[str] = None, +) -> bool: +''' +specify_signature_after = '''def specify_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + title: Optional[str] = None, + body: Optional[str] = None, + assignee: Optional[str] = None, + author: Optional[str] = None, + require_no_runs: bool = False, +) -> bool: +''' +db = replace_once( + db, + specify_signature_before, + specify_signature_after, + "transactional specify signature", +) + +specify_guard_before = ''' if existing is None: + return False + sets: list[str] = ["status = 'todo'"] +''' +specify_guard_after = ''' if existing is None: + return False + if require_no_runs and conn.execute( + "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", + (task_id,), + ).fetchone() is not None: + return False + sets: list[str] = ["status = 'todo'"] +''' +db = replace_once( + db, + specify_guard_before, + specify_guard_after, + "transactional specify execution-history guard", +) + +decompose_signature_before = '''def decompose_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + root_assignee: Optional[str], + children: list[dict], + author: Optional[str] = None, + auto_promote: bool = True, +) -> Optional[list[str]]: +''' +decompose_signature_after = '''def decompose_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + root_assignee: Optional[str], + children: list[dict], + author: Optional[str] = None, + auto_promote: bool = True, + require_no_runs: bool = False, +) -> Optional[list[str]]: +''' +db = replace_once( + db, + decompose_signature_before, + decompose_signature_after, + "transactional decomposition signature", +) + +decompose_guard_before = ''' if root_row["status"] != "triage": + return None + tenant = root_row["tenant"] +''' +decompose_guard_after = ''' if root_row["status"] != "triage": + return None + if require_no_runs and conn.execute( + "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1", + (task_id,), + ).fetchone() is not None: + return None + tenant = root_row["tenant"] +''' +db = replace_once( + db, + decompose_guard_before, + decompose_guard_after, + "transactional decomposition execution-history guard", +) +db_path.write_text(db, encoding="utf-8") + +decompose_path = source_root / "hermes_cli/kanban_decompose.py" +decompose = decompose_path.read_text(encoding="utf-8") + +helper_anchor = ''' if chosen not in valid_names: + return default_assignee + return chosen + + +''' +helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool: + """Return whether a task has ever entered a worker run.""" + with kb.connect_closing() as conn: + return bool(kb.list_runs(conn, task_id)) + + +''' +decompose = replace_once( + decompose, + helper_anchor, + helper_replacement, + "auto-decompose execution-history helper", +) + +signature_before = '''def decompose_task( + task_id: str, + *, + author: Optional[str] = None, + timeout: Optional[int] = None, +) -> DecomposeOutcome: +''' +signature_after = '''def decompose_task( + task_id: str, + *, + author: Optional[str] = None, + timeout: Optional[int] = None, + automatic: bool = False, +) -> DecomposeOutcome: +''' +decompose = replace_once( + decompose, + signature_before, + signature_after, + "auto-decompose function signature", +) + +status_before = ''' if task.status != "triage": + return DecomposeOutcome( + task_id, False, f"task is not in triage (status={task.status!r})" + ) + + cfg = _load_config() +''' +status_after = ''' if task.status != "triage": + return DecomposeOutcome( + task_id, False, f"task is not in triage (status={task.status!r})" + ) + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task has execution history and requires deliberate manual triage", + ) + + cfg = _load_config() +''' +decompose = replace_once( + decompose, + status_before, + status_after, + "auto-decompose preflight guard", +) + +single_before = ''' author=audit_author, + ) + if not ok: +''' +single_after = ''' author=audit_author, + require_no_runs=automatic, + ) + if not ok: + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) +''' +decompose = replace_once( + decompose, + single_before, + single_after, + "auto-decompose single-task commit guard", +) + +fanout_before = ''' author=audit_author, + auto_promote=auto_promote, + ) +''' +fanout_after = ''' author=audit_author, + auto_promote=auto_promote, + require_no_runs=automatic, + ) +''' +decompose = replace_once( + decompose, + fanout_before, + fanout_after, + "auto-decompose transactional fanout guard", +) + +fanout_outcome_before = ''' if child_ids is None: + return DecomposeOutcome( + task_id, False, "task moved out of triage before decomposition", + ) +''' +fanout_outcome_after = ''' if child_ids is None: + if automatic and _has_execution_history(task_id): + return DecomposeOutcome( + task_id, + False, + "task gained execution history and requires deliberate manual triage", + ) + return DecomposeOutcome( + task_id, False, "task moved out of triage before decomposition", + ) +''' +decompose = replace_once( + decompose, + fanout_outcome_before, + fanout_outcome_after, + "auto-decompose fanout rejection reason", +) +decompose_path.write_text(decompose, encoding="utf-8") + + +watcher_path = source_root / "gateway/kanban_watchers.py" +watcher = watcher_path.read_text(encoding="utf-8") +call_before = ''' outcome = _decomp.decompose_task( + tid, author="auto-decomposer", + ) +''' +call_after = ''' outcome = _decomp.decompose_task( + tid, + author="auto-decomposer", + automatic=True, + ) +''' +watcher = replace_once( + watcher, + call_before, + call_after, + "gateway automatic-decomposition call", +) +watcher_path.write_text(watcher, encoding="utf-8") diff --git a/dockerfiles/patch_hermes_run_safety.py b/dockerfiles/patch_hermes_run_safety.py new file mode 100644 index 00000000..09eecb2f --- /dev/null +++ b/dockerfiles/patch_hermes_run_safety.py @@ -0,0 +1,480 @@ +"""Patch exact-run completion, provenance, and reclaim transactions.""" + +import os +from pathlib import Path + +from hermes_execution_patch_support import replace_once + + + +source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) +db_path = source_root / "hermes_cli/kanban_db.py" +db = db_path.read_text(encoding="utf-8") + +complete_signature_before = '''def complete_task( + conn: sqlite3.Connection, + task_id: str, + *, + result: Optional[str] = None, + summary: Optional[str] = None, + metadata: Optional[dict] = None, + created_cards: Optional[Iterable[str]] = None, + expected_run_id: Optional[int] = None, +) -> bool: +''' +complete_signature_after = '''def complete_task( + conn: sqlite3.Connection, + task_id: str, + *, + result: Optional[str] = None, + summary: Optional[str] = None, + metadata: Optional[dict] = None, + created_cards: Optional[Iterable[str]] = None, + expected_run_id: Optional[int] = None, + replay_ended_run_id: Optional[int] = None, +) -> bool: +''' +db = replace_once( + db, + complete_signature_before, + complete_signature_after, + "exact ended-run completion signature", +) + +complete_branch_before = ''' with write_txn(conn): + if expected_run_id is None: + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('running', 'ready', 'blocked', 'scheduled') + """, + (result, now, task_id), + ) + else: +''' +complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the + # accepted run. Verify that it is still the latest run in this same + # transaction; a replacement attempt must make the replay fail closed. + with write_txn(conn): + replayed_run_id = None + if expected_run_id is not None and replay_ended_run_id is not None: + raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive") + if replay_ended_run_id is not None: + replayed_run_id = int(replay_ended_run_id) + # Replay is permitted only while the exact run-ending transition + # remains the latest status mutation. Comments and diagnostics do + # not invalidate durable accepted work, but an unblock, explicit + # triage specification, decomposition, reclaim, or newer attempt + # does. BEGIN IMMEDIATE keeps this lineage check and the guarded + # task/run updates in one transaction. + last_transition = conn.execute( + """ + SELECT kind, run_id + FROM task_events + WHERE task_id = ? + AND kind IN ( + 'archived', 'block_loop_detected', 'blocked', 'claimed', + 'completed', 'decomposed', 'dependency_wait', 'gave_up', + 'promoted', 'promoted_manual', 'reclaimed', 'scheduled', + 'specified', 'stale', 'timed_out', 'unblocked' + ) + ORDER BY id DESC + LIMIT 1 + """, + (task_id,), + ).fetchone() + if ( + last_transition is None + or last_transition["run_id"] != replayed_run_id + or last_transition["kind"] not in { + "block_loop_detected", + "blocked", + "reclaimed", + "stale", + "timed_out", + "gave_up", + } + ): + return False + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('ready', 'blocked', 'triage') + AND current_run_id IS NULL + AND ? = ( + SELECT id FROM task_runs + WHERE task_id = ? + ORDER BY id DESC + LIMIT 1 + ) + AND EXISTS ( + SELECT 1 FROM task_runs + WHERE id = ? + AND task_id = ? + AND ended_at IS NOT NULL + ) + """, + ( + result, + now, + task_id, + replayed_run_id, + task_id, + replayed_run_id, + task_id, + ), + ) + elif expected_run_id is None: + cur = conn.execute( + """ + UPDATE tasks + SET status = 'done', + result = ?, + completed_at = ?, + claim_lock = NULL, + claim_expires= NULL, + worker_pid = NULL, + block_kind = NULL, + block_recurrences = 0 + WHERE id = ? + AND status IN ('running', 'ready', 'blocked', 'scheduled') + """, + (result, now, task_id), + ) + else: +''' +db = replace_once( + db, + complete_branch_before, + complete_branch_after, + "transactional exact ended-run completion", +) + +run_completion_before = ''' run_id = _end_run( + conn, task_id, + outcome="completed", status="done", + summary=summary if summary is not None else result, + metadata=metadata, + ) + # If complete_task was called on a never-claimed task (ready or + # blocked → done with no run in flight), synthesize a + # zero-duration run so the handoff fields are persisted in + # attempt history instead of silently lost. + if run_id is None and (summary or metadata or result): + run_id = _synthesize_ended_run( + conn, task_id, + outcome="completed", + summary=summary if summary is not None else result, + metadata=metadata, + ) +''' +run_completion_after = ''' if replayed_run_id is None: + run_id = _end_run( + conn, task_id, + outcome="completed", status="done", + summary=summary if summary is not None else result, + metadata=metadata, + ) + # If complete_task was called on a never-claimed task (ready or + # blocked → done with no run in flight), synthesize a + # zero-duration run so the handoff fields are persisted in + # attempt history instead of silently lost. + if run_id is None and (summary or metadata or result): + run_id = _synthesize_ended_run( + conn, task_id, + outcome="completed", + summary=summary if summary is not None else result, + metadata=metadata, + ) + else: + run_id = replayed_run_id + replayed = conn.execute( + """ + UPDATE task_runs + SET status = 'done', + outcome = 'completed', + summary = ?, + error = NULL, + metadata = ?, + claim_lock = NULL, + claim_expires = NULL, + worker_pid = NULL + WHERE id = ? + AND task_id = ? + AND ended_at IS NOT NULL + """, + ( + summary if summary is not None else result, + json.dumps(metadata, ensure_ascii=False) if metadata else None, + run_id, + task_id, + ), + ) + if replayed.rowcount != 1: + raise RuntimeError("exact ended-run completion lost its run identity") + conn.execute( + "UPDATE tasks SET completed_run_id = ? WHERE id = ?", + (run_id, task_id), + ) +''' +db = replace_once( + db, + run_completion_before, + run_completion_after, + "exact completion run provenance", +) + +reclaim_before = '''def reclaim_task( + conn: sqlite3.Connection, + task_id: str, + *, + reason: Optional[str] = None, + signal_fn=None, +) -> bool: + """Operator-driven reclaim: release the claim and reset to ``ready``. + + Unlike :func:`release_stale_claims` which only acts on tasks whose + ``claim_expires`` has passed, this function reclaims immediately + regardless of TTL. Intended for the dashboard/CLI recovery flow + when an operator wants to abort a running worker without waiting + for the TTL to expire (e.g. after seeing a hallucination warning). + + Returns True if a reclaim happened, False if the task isn't in a + reclaimable state (not running, or doesn't exist). + """ + row = conn.execute( + "SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?", + (task_id,), + ).fetchone() + if not row: + return False + if row["status"] != "running" and row["claim_lock"] is None: + # Nothing to reclaim — already ready / blocked / done. + return False + prev_lock = row["claim_lock"] + termination = _terminate_reclaimed_worker( + row["worker_pid"], prev_lock, signal_fn=signal_fn, + ) + with write_txn(conn): + cur = conn.execute( + "UPDATE tasks SET status = 'ready', claim_lock = NULL, " + "claim_expires = NULL, worker_pid = NULL " + "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " + "AND claim_lock IS ?", + (task_id, prev_lock), + ) + if cur.rowcount != 1: + return False + run_id = _end_run( + conn, task_id, + outcome="reclaimed", status="reclaimed", + error=( + f"manual_reclaim: {reason}" if reason + else f"manual_reclaim lock={prev_lock}" + ), + metadata=termination, + ) + payload = { + "manual": True, + "reason": reason, + "prev_lock": prev_lock, + } + payload.update(termination) + _append_event( + conn, task_id, "reclaimed", + payload, + run_id=run_id, + ) + # Operator intervention — they've looked at the task, so the + # consecutive-failures counter is now stale. Give the next retry + # a fresh budget. (_clear_failure_counter opens its own write_txn, + # so it runs after the enclosing one commits.) + _clear_failure_counter(conn, task_id) + return True +''' +reclaim_after = '''def reclaim_task( + conn: sqlite3.Connection, + task_id: str, + *, + reason: Optional[str] = None, + signal_fn=None, + expected_run_id: Optional[int] = None, +) -> bool: + """Operator-driven reclaim: release the claim and reset to ``ready``. + + Unlike :func:`release_stale_claims` which only acts on tasks whose + ``claim_expires`` has passed, this function reclaims immediately + regardless of TTL. Intended for the dashboard/CLI recovery flow + when an operator wants to abort a running worker without waiting + for the TTL to expire (e.g. after seeing a hallucination warning). + + When ``expected_run_id`` is supplied, the run identity is checked after + ``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run + is therefore neither signalled nor reclaimed by stale recovery evidence. + + Returns True if a reclaim happened, False if the task isn't in a + reclaimable state (not running, or doesn't exist), or its run changed. + """ + with write_txn(conn): + row = conn.execute( + "SELECT status, claim_lock, worker_pid, current_run_id " + "FROM tasks WHERE id = ?", + (task_id,), + ).fetchone() + if not row: + return False + if expected_run_id is not None and row["current_run_id"] != expected_run_id: + return False + if row["status"] != "running" and row["claim_lock"] is None: + # Nothing to reclaim — already ready / blocked / done. + return False + prev_lock = row["claim_lock"] + termination = _terminate_reclaimed_worker( + row["worker_pid"], prev_lock, signal_fn=signal_fn, + ) + if expected_run_id is None: + cur = conn.execute( + "UPDATE tasks SET status = 'ready', claim_lock = NULL, " + "claim_expires = NULL, worker_pid = NULL " + "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " + "AND claim_lock IS ?", + (task_id, prev_lock), + ) + else: + cur = conn.execute( + "UPDATE tasks SET status = 'ready', claim_lock = NULL, " + "claim_expires = NULL, worker_pid = NULL " + "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " + "AND claim_lock IS ? AND current_run_id = ?", + (task_id, prev_lock, int(expected_run_id)), + ) + if cur.rowcount != 1: + return False + run_id = _end_run( + conn, task_id, + outcome="reclaimed", status="reclaimed", + error=( + f"manual_reclaim: {reason}" if reason + else f"manual_reclaim lock={prev_lock}" + ), + metadata=termination, + ) + payload = { + "manual": True, + "reason": reason, + "prev_lock": prev_lock, + } + payload.update(termination) + _append_event( + conn, task_id, "reclaimed", + payload, + run_id=run_id, + ) + # Operator intervention — they've looked at the task, so the + # consecutive-failures counter is now stale. Give the next retry + # a fresh budget. (_clear_failure_counter opens its own write_txn, + # so it runs after the enclosing one commits.) + _clear_failure_counter(conn, task_id) + return True +''' +db = replace_once( + db, + reclaim_before, + reclaim_after, + "transactional exact-run reclaim", +) + +task_field_before = ''' current_run_id: Optional[int] = None + workflow_template_id: Optional[str] = None +''' +task_field_after = ''' current_run_id: Optional[int] = None + # Durable provenance for the run that authoritatively completed this task. + completed_run_id: Optional[int] = None + workflow_template_id: Optional[str] = None +''' +db = replace_once( + db, + task_field_before, + task_field_after, + "completed-run task field", +) + +task_row_before = ''' current_run_id=( + row["current_run_id"] if "current_run_id" in keys else None + ), + workflow_template_id=( +''' +task_row_after = ''' current_run_id=( + row["current_run_id"] if "current_run_id" in keys else None + ), + completed_run_id=( + row["completed_run_id"] if "completed_run_id" in keys else None + ), + workflow_template_id=( +''' +db = replace_once( + db, + task_row_before, + task_row_after, + "completed-run row mapping", +) + +task_schema_before = ''' -- run is in-flight). Denormalised for cheap reads. + current_run_id INTEGER, + -- Forward-compat for v2 workflow routing. In v1 the kernel writes +''' +task_schema_after = ''' -- run is in-flight). Denormalised for cheap reads. + current_run_id INTEGER, + -- Immutable provenance for the task result's authoritative run. + completed_run_id INTEGER, + -- Forward-compat for v2 workflow routing. In v1 the kernel writes +''' +db = replace_once( + db, + task_schema_before, + task_schema_after, + "completed-run schema", +) + +task_migration_before = ''' if "current_run_id" not in cols: + _add_column_if_missing( + conn, "tasks", "current_run_id", "current_run_id INTEGER" + ) + if "workflow_template_id" not in cols: +''' +task_migration_after = ''' if "current_run_id" not in cols: + _add_column_if_missing( + conn, "tasks", "current_run_id", "current_run_id INTEGER" + ) + if "completed_run_id" not in cols: + _add_column_if_missing( + conn, "tasks", "completed_run_id", "completed_run_id INTEGER" + ) + if "workflow_template_id" not in cols: +''' +db = replace_once( + db, + task_migration_before, + task_migration_after, + "completed-run migration", +) + +db_path.write_text(db, encoding="utf-8") diff --git a/scripts/tests/test_dashboard_render_support.py b/scripts/tests/test_dashboard_render_support.py new file mode 100644 index 00000000..a073eb03 --- /dev/null +++ b/scripts/tests/test_dashboard_render_support.py @@ -0,0 +1,25 @@ +import importlib.util + +import pathlib + +def load_module(): + path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py" + spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + +def flatten_panels(panels): + flat = [] + for panel in panels: + flat.append(panel) + flat.extend(panel.get("panels", [])) + return flat + +__all__ = ( + "flatten_panels", + "importlib", + "load_module", + "pathlib", +) diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_jobs.py similarity index 54% rename from scripts/tests/test_dashboards_render_atlas.py rename to scripts/tests/test_dashboards_render_jobs.py index bba16cdf..207c30b3 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_jobs.py @@ -1,241 +1,11 @@ -import importlib.util -import pathlib +"""Atlas testing and jobs dashboard rendering contracts.""" +from __future__ import annotations -def load_module(): - path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py" - spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -def flatten_panels(panels): - flat = [] - for panel in panels: - flat.append(panel) - flat.extend(panel.get("panels", [])) - return flat - - -def test_table_panel_options_and_filterable(): - mod = load_module() - panel = mod.table_panel( - 1, - "test", - "metric", - {"h": 1, "w": 1, "x": 0, "y": 0}, - unit="percent", - transformations=[{"id": "labelsToFields", "options": {}}], - instant=True, - options={"showColumnFilters": False}, - filterable=False, - footer={"show": False, "fields": "", "calcs": []}, - format="table", - ) - assert panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False - assert panel["options"]["showHeader"] is True - assert panel["targets"][0]["format"] == "table" - - -def test_node_filter_and_expr_helpers(): - mod = load_module() - expr = mod.node_filter("titan-.*") - assert "label_replace" in expr - cpu_expr = mod.node_cpu_expr("titan-.*") - mem_expr = mod.node_mem_expr("titan-.*") - assert "node_cpu_seconds_total" in cpu_expr - assert "clamp_max(clamp_min(" in cpu_expr - assert "* 100, 0), 100)" in cpu_expr - assert "node_memory_MemAvailable_bytes" in mem_expr - - -def test_overview_availability_panel_uses_recorded_365d_rollup(): - mod = load_module() - dashboard = mod.build_overview() - panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) - - assert panel["title"] == "Atlas Availability (365d)" - availability_expr = panel["targets"][0]["expr"] - assert ( - 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])' - in availability_expr - ) - assert 'code=~"5.."' in availability_expr - assert 'code=~"[1-5].."' in availability_expr - assert "atlas:availability:failures_1d" not in availability_expr - assert "atlas:availability:requests_1d" not in availability_expr - assert "sum_over_time" not in availability_expr - assert "kube_node_status_condition" not in availability_expr - assert "kube_deployment_status_replicas_available" not in availability_expr - assert panel["targets"][0]["instant"] is True - assert "Every server-side 5xx" in panel["description"] - assert "Replica counts, Grafana health" in panel["description"] - assert "daily rollup job publishes one annual sample" in panel["description"] - assert "keeps it for up to 48 hours" in panel["description"] - - -def test_overview_uses_readable_quality_power_and_gitops_panels(): - mod = load_module() - dashboard = mod.build_overview() - panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} - - assert dashboard["links"] == [ - {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True} - ] - assert "atlas-jobs" not in repr(dashboard) - assert "Platform Test Success Rate" not in panels_by_title - assert panels_by_title["Test Category Health"]["type"] == "state-timeline" - assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13} - assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}" - assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series" - assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True - assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"] - assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"] - assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"] - assert panels_by_title["Test Category Health"]["timeFrom"] == "24h" - assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"] - assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX - assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX - assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7} - assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7} - assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3 - assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3 - assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom" - assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list" - assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line" - assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22 - assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"]) - ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"] - ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides} - assert ups_override_by_name["Pyrphoros"]["properties"] == [ - {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}}, - ] - assert ups_override_by_name["Statera"]["properties"] == [ - {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}}, - ] - assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" - assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom" - assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list" - assert "Fan History (0-10)" not in panels_by_title - assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline" - assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13} - assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10 - assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}" - fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"] - assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - assert fan_steps[0]["color"] == "#8f1d1d" - assert fan_steps[5]["color"] == "#d4b106" - assert fan_steps[-1]["color"] == "#1f60c4" - fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"] - assert fan_mappings["0"]["text"] == "Off" - assert fan_mappings["10"]["text"] == "10" - assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"] - assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list" - assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False - assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False - assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto" - - assert panels_by_title["Flux Source"]["type"] == "stat" - assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7} - assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}" - assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9} - assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"] - assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11} - assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15} - suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"] - assert "> bool 0" in suites_reporting_expr - assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr - assert "bstein-home" not in suites_reporting_expr - assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"] - assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17} - assert "LOC Clean Suites" not in panels_by_title - assert panels_by_title["GitOps Health"]["type"] == "state-timeline" - assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7} - gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"] - assert "Kustomizations Not Suspended" in gitops_expr - assert "HelmReleases Not Suspended" in gitops_expr - assert panels_by_title["Test Category Health"]["type"] == "state-timeline" - assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False - assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False - assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto" - assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9 - assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}" - assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"] - assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"]) - - pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"] - assert "backup-telemetry-missing" in pvc_backup_expr - assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr - - gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"] - assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr - assert "nvidia_gpu_device_utilization_percent" in gpu_expr - assert "jetson_gr3d_active_seconds_total" in gpu_expr - assert "increase" in gpu_expr - assert "sum_over_time" not in gpu_expr - assert "count_over_time" not in gpu_expr - assert "avg_over_time" in gpu_expr - assert "$__range" in gpu_expr - assert "$__range_s" in gpu_expr - assert "sum by (namespace)" in gpu_expr - assert 'namespace", "shared"' not in gpu_expr - assert "kube_pod_container_resource_requests" in gpu_expr - assert mod.GPU_RESOURCE_REGEX in gpu_expr - assert '"gpu_source", "nvidia"' in gpu_expr - assert '"gpu_source", "jetson"' in gpu_expr - assert "100 *" in gpu_expr - assert "100 -" not in gpu_expr - assert 'namespace", "unattributed"' in gpu_expr - assert 'namespace", "idle"' in gpu_expr - assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True - assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"] - assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"] - assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"] - - -def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families(): - mod = load_module() - dashboard = mod.build_gpu_dashboard() - panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} - - node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"] - assert "nvidia_gpu_device_utilization_percent" in node_expr - assert "DCGM_FI_DEV_GPU_UTIL" in node_expr - assert "jetson_gr3d_freq_percent" in node_expr - assert "unless on(node)" in node_expr - assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"] - - process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"] - assert "nvidia_process_gpu_sm_util_percent" in process_expr - assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr - - -def test_overview_and_testing_panels_all_have_concise_descriptions(): - mod = load_module() - - for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]: - panels = flatten_panels(dashboard["panels"]) - assert all(panel.get("description") for panel in panels if panel["type"] != "row") - assert "85.7" not in repr(dashboard) - - -def test_render_configmap_writes(tmp_path): - mod = load_module() - mod.DASHBOARD_DIR = tmp_path / "dash" - mod.ROOT = tmp_path - uid = "atlas-test" - info = {"configmap": tmp_path / "cm.yaml"} - data = {"title": "Atlas Test"} - mod.write_json(uid, data) - mod.render_configmap(uid, info) - json_path = mod.DASHBOARD_DIR / f"{uid}.json" - assert json_path.exists() - content = (tmp_path / "cm.yaml").read_text() - assert "kind: ConfigMap" in content - assert f"{uid}.json" in content +from test_dashboard_render_support import ( + flatten_panels, + load_module, +) def test_testing_suite_variable_uses_configured_suite_values_only(): @@ -257,7 +27,6 @@ def test_testing_suite_variable_uses_configured_suite_values_only(): option["value"] in out_of_scope_names for option in variable["options"] ) - def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal(): mod = load_module() jobs = mod.build_jobs_dashboard() @@ -269,7 +38,6 @@ def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal(): assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER assert testing["editable"] is False - def test_jobs_dashboard_separates_current_gate_health_from_reliability(): mod = load_module() dashboard = mod.build_jobs_dashboard() @@ -395,7 +163,6 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability(): assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"] - def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): mod = load_module() dashboard = mod.build_jobs_dashboard() @@ -417,7 +184,6 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): assert {"color": "dark-yellow", "value": 93} in threshold_steps assert {"color": "dark-blue", "value": 100} in threshold_steps - def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): mod = load_module() dashboard = mod.build_jobs_dashboard() diff --git a/scripts/tests/test_dashboards_render_overview.py b/scripts/tests/test_dashboards_render_overview.py new file mode 100644 index 00000000..59eadb5c --- /dev/null +++ b/scripts/tests/test_dashboards_render_overview.py @@ -0,0 +1,221 @@ +"""Atlas overview dashboard rendering contracts.""" + +from __future__ import annotations + +from test_dashboard_render_support import ( + flatten_panels, + load_module, +) + + +def test_table_panel_options_and_filterable(): + mod = load_module() + panel = mod.table_panel( + 1, + "test", + "metric", + {"h": 1, "w": 1, "x": 0, "y": 0}, + unit="percent", + transformations=[{"id": "labelsToFields", "options": {}}], + instant=True, + options={"showColumnFilters": False}, + filterable=False, + footer={"show": False, "fields": "", "calcs": []}, + format="table", + ) + assert panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False + assert panel["options"]["showHeader"] is True + assert panel["targets"][0]["format"] == "table" + +def test_node_filter_and_expr_helpers(): + mod = load_module() + expr = mod.node_filter("titan-.*") + assert "label_replace" in expr + cpu_expr = mod.node_cpu_expr("titan-.*") + mem_expr = mod.node_mem_expr("titan-.*") + assert "node_cpu_seconds_total" in cpu_expr + assert "clamp_max(clamp_min(" in cpu_expr + assert "* 100, 0), 100)" in cpu_expr + assert "node_memory_MemAvailable_bytes" in mem_expr + +def test_overview_availability_panel_uses_recorded_365d_rollup(): + mod = load_module() + dashboard = mod.build_overview() + panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) + + assert panel["title"] == "Atlas Availability (365d)" + availability_expr = panel["targets"][0]["expr"] + assert ( + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])' + in availability_expr + ) + assert 'code=~"5.."' in availability_expr + assert 'code=~"[1-5].."' in availability_expr + assert "atlas:availability:failures_1d" not in availability_expr + assert "atlas:availability:requests_1d" not in availability_expr + assert "sum_over_time" not in availability_expr + assert "kube_node_status_condition" not in availability_expr + assert "kube_deployment_status_replicas_available" not in availability_expr + assert panel["targets"][0]["instant"] is True + assert "Every server-side 5xx" in panel["description"] + assert "Replica counts, Grafana health" in panel["description"] + assert "daily rollup job publishes one annual sample" in panel["description"] + assert "keeps it for up to 48 hours" in panel["description"] + +def test_overview_uses_readable_quality_power_and_gitops_panels(): + mod = load_module() + dashboard = mod.build_overview() + panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} + + assert dashboard["links"] == [ + {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True} + ] + assert "atlas-jobs" not in repr(dashboard) + assert "Platform Test Success Rate" not in panels_by_title + assert panels_by_title["Test Category Health"]["type"] == "state-timeline" + assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13} + assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}" + assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series" + assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True + assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"] + assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"] + assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"] + assert panels_by_title["Test Category Health"]["timeFrom"] == "24h" + assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"] + assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX + assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX + assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7} + assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7} + assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3 + assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3 + assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom" + assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list" + assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line" + assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22 + assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"]) + ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"] + ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides} + assert ups_override_by_name["Pyrphoros"]["properties"] == [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}}, + ] + assert ups_override_by_name["Statera"]["properties"] == [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}}, + ] + assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" + assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom" + assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list" + assert "Fan History (0-10)" not in panels_by_title + assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline" + assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13} + assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10 + assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}" + fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"] + assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + assert fan_steps[0]["color"] == "#8f1d1d" + assert fan_steps[5]["color"] == "#d4b106" + assert fan_steps[-1]["color"] == "#1f60c4" + fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"] + assert fan_mappings["0"]["text"] == "Off" + assert fan_mappings["10"]["text"] == "10" + assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"] + assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list" + assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False + assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False + assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto" + + assert panels_by_title["Flux Source"]["type"] == "stat" + assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7} + assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}" + assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9} + assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"] + assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11} + assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15} + suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"] + assert "> bool 0" in suites_reporting_expr + assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr + assert "bstein-home" not in suites_reporting_expr + assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"] + assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17} + assert "LOC Clean Suites" not in panels_by_title + assert panels_by_title["GitOps Health"]["type"] == "state-timeline" + assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7} + gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"] + assert "Kustomizations Not Suspended" in gitops_expr + assert "HelmReleases Not Suspended" in gitops_expr + assert panels_by_title["Test Category Health"]["type"] == "state-timeline" + assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False + assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False + assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto" + assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9 + assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}" + assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"] + assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"]) + + pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"] + assert "backup-telemetry-missing" in pvc_backup_expr + assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr + + gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"] + assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr + assert "nvidia_gpu_device_utilization_percent" in gpu_expr + assert "jetson_gr3d_active_seconds_total" in gpu_expr + assert "increase" in gpu_expr + assert "sum_over_time" not in gpu_expr + assert "count_over_time" not in gpu_expr + assert "avg_over_time" in gpu_expr + assert "$__range" in gpu_expr + assert "$__range_s" in gpu_expr + assert "sum by (namespace)" in gpu_expr + assert 'namespace", "shared"' not in gpu_expr + assert "kube_pod_container_resource_requests" in gpu_expr + assert mod.GPU_RESOURCE_REGEX in gpu_expr + assert '"gpu_source", "nvidia"' in gpu_expr + assert '"gpu_source", "jetson"' in gpu_expr + assert "100 *" in gpu_expr + assert "100 -" not in gpu_expr + assert 'namespace", "unattributed"' in gpu_expr + assert 'namespace", "idle"' in gpu_expr + assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True + assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"] + +def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families(): + mod = load_module() + dashboard = mod.build_gpu_dashboard() + panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} + + node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"] + assert "nvidia_gpu_device_utilization_percent" in node_expr + assert "DCGM_FI_DEV_GPU_UTIL" in node_expr + assert "jetson_gr3d_freq_percent" in node_expr + assert "unless on(node)" in node_expr + assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"] + + process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"] + assert "nvidia_process_gpu_sm_util_percent" in process_expr + assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr + +def test_overview_and_testing_panels_all_have_concise_descriptions(): + mod = load_module() + + for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]: + panels = flatten_panels(dashboard["panels"]) + assert all(panel.get("description") for panel in panels if panel["type"] != "row") + assert "85.7" not in repr(dashboard) + +def test_render_configmap_writes(tmp_path): + mod = load_module() + mod.DASHBOARD_DIR = tmp_path / "dash" + mod.ROOT = tmp_path + uid = "atlas-test" + info = {"configmap": tmp_path / "cm.yaml"} + data = {"title": "Atlas Test"} + mod.write_json(uid, data) + mod.render_configmap(uid, info) + json_path = mod.DASHBOARD_DIR / f"{uid}.json" + assert json_path.exists() + content = (tmp_path / "cm.yaml").read_text() + assert "kind: ConfigMap" in content + assert f"{uid}.json" in content diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..706a56b0 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -61,7 +61,21 @@ configMapGenerator: - ai_usage_exporter.py=scripts/ai_usage_exporter.py - claude=scripts/claude - claude_command_policy.py=scripts/claude_command_policy.py + - cli_lane_board.py=scripts/cli_lane_board.py + - cli_lane_config.py=scripts/cli_lane_config.py + - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py + - cli_lane_evidence.py=scripts/cli_lane_evidence.py + - cli_lane_execution.py=scripts/cli_lane_execution.py + - cli_lane_files.py=scripts/cli_lane_files.py + - cli_lane_finalization.py=scripts/cli_lane_finalization.py - cli_lane_goal.py=scripts/cli_lane_goal.py + - cli_lane_prompt.py=scripts/cli_lane_prompt.py + - cli_lane_provider.py=scripts/cli_lane_provider.py + - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py + - cli_lane_records.py=scripts/cli_lane_records.py + - cli_lane_recovery.py=scripts/cli_lane_recovery.py + - cli_lane_retention.py=scripts/cli_lane_retention.py + - cli_lane_routing.py=scripts/cli_lane_routing.py - cli_lane_runner.py=scripts/cli_lane_runner.py - codex=scripts/codex - configure_agent_clients.py=scripts/configure_agent_clients.py diff --git a/services/hermes/scripts/cli_lane_board.py b/services/hermes/scripts/cli_lane_board.py new file mode 100644 index 00000000..fd173756 --- /dev/null +++ b/services/hermes/scripts/cli_lane_board.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Bounded Kanban access and workspace resolution for Hermes CLI workers.""" + +from __future__ import annotations + +import json +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any, Callable + +from cli_lane_config import ( + BOARD_CORRUPTION_ERRORS, + EXTERNAL_PREFIX, + KANBAN_STORAGE_ATTEMPTS, + WORKTREE_LOCK, +) + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + +def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str: + value = kanban_db.build_worker_context(conn, task_id) + if isinstance(value, str): + return value + return json.dumps(value, indent=2, default=str) + +def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path: + # External coding lanes always receive their own linked worktree. A task + # that does not resolve to a Git repository is blocked rather than sharing + # a mutable checkout with another unattended worker. + # Git worktree creation updates common-repository metadata. Serialize that + # short materialization step while allowing the provider workers themselves + # to run concurrently in independent worktrees. + with WORKTREE_LOCK: + value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board) + kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name) + return Path(value).resolve() + +def _record_board_access_error(board: str, error: Exception) -> None: + """Report one board access failure once while allowing other lanes to run.""" + failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access" + detail = f"{failure_kind} {type(error).__name__}: {error}" + if BOARD_CORRUPTION_ERRORS.get(board) == detail: + return + print( + f"temporarily skipping Kanban board {board!r}: {detail}", + file=sys.stderr, + flush=True, + ) + BOARD_CORRUPTION_ERRORS[board] = detail + +def _board_call( + kanban_db: Any, + board: str, + operation: Callable[[Any], Any], +) -> Any: + """Run one bounded Kanban operation on a fresh, promptly closed connection.""" + last_storage_error: Exception | None = None + for attempt in range(KANBAN_STORAGE_ATTEMPTS): + conn = None + try: + with kanban_db.scoped_current_board(board): + conn = kanban_db.connect(board=board) + result = operation(conn) + BOARD_CORRUPTION_ERRORS.pop(board, None) + return result + except (OSError, sqlite3.Error) as error: + last_storage_error = error + _record_board_access_error(board, error) + if attempt + 1 < KANBAN_STORAGE_ATTEMPTS: + time.sleep(0.2 * (attempt + 1)) + finally: + if conn is not None: + conn.close() + assert last_storage_error is not None + raise last_storage_error + +def _external(task: Any) -> bool: + return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX) diff --git a/services/hermes/scripts/cli_lane_config.py b/services/hermes/scripts/cli_lane_config.py new file mode 100644 index 00000000..dcf29faa --- /dev/null +++ b/services/hermes/scripts/cli_lane_config.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Shared immutable configuration and result types for Hermes CLI lanes.""" + +from __future__ import annotations + +import os +import re +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import cli_lane_goal + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +SWITCHYARD_URL = os.environ.get( + "HERMES_SWITCHYARD_URL", + "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions", +) +STATE_ROOT = DATA_ROOT / "cli-lanes" +CODEX_BIN = DATA_ROOT / "tools/bin/codex" +CLAUDE_BIN = DATA_ROOT / "tools/bin/claude" +CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json" +RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json" +EFFORTS = ("low", "medium", "high", "xhigh") +EXTERNAL_PREFIX = "cli-" +DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60 +DEFAULT_MAX_RUNTIME = 12 * 60 * 60 +HEARTBEAT_SECONDS = 20 +PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 +PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60 +KANBAN_STORAGE_ATTEMPTS = 5 +ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60 +ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60 +ARTIFACT_RETENTION_COUNT = 256 +ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024 +MAX_TERMINAL_RECORD_BYTES = 1024 * 1024 +QUARANTINE_HASH_BYTES = 64 * 1024 +SQLITE_RUN_ID_MAX = 2**63 - 1 +PROVIDER_HEALTH_PATHS = { + "codex": DATA_ROOT / "provider-health/codex.json", + "claude": DATA_ROOT / "provider-health/claude.json", +} +WORKTREE_LOCK = threading.Lock() +BOARD_CORRUPTION_ERRORS: dict[str, str] = {} +LAST_ARTIFACT_GC = 0.0 +CAPACITY_PATTERN = re.compile( + r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", + re.I, +) +NO_CLAUDE_SESSION = "No conversation found with session ID:" +CLAUDE_SESSION_COLLISION = "Session ID already in use" +NO_CODEX_THREAD = "no rollout found for thread id" + + +RESULT_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": [ + "status", + "summary", + "changed_files", + "tests_run", + "artifacts", + "findings", + "blockers", + ], + "properties": { + "status": { + "type": "string", + "enum": sorted(cli_lane_goal.RESULT_STATUSES), + }, + "summary": {"type": "string"}, + "changed_files": {"type": "array", "items": {"type": "string"}}, + "tests_run": {"type": "array", "items": {"type": "string"}}, + "artifacts": {"type": "array", "items": {"type": "string"}}, + "findings": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Defects, risks, or observations discovered by a review or diagnosis. " + "Findings do not prevent the assigned review or diagnosis from completing." + ), + }, + "blockers": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Concrete obstacles that prevent completion of the assigned task itself. " + "This must be empty when status is completed; review findings belong in findings." + ), + }, + }, +} + + +@dataclass(frozen=True) +class Route: + """One provider/model/effort decision for an observable worker boundary.""" + + provider: str + model: str + effort: str + profile: str + classifier: str + reason: str + latency_ms: int + fallback_chain: tuple[str, ...] + + +@dataclass +class ProcessResult: + """Captured outcome from one provider CLI invocation.""" + + returncode: int + output: str + structured: dict[str, Any] | None + capacity_failure: bool + + +class TerminalFinalizationPending(RuntimeError): + """An accepted worker result is durable but not committed to Kanban yet.""" + + +def canonical_run_id(value: object) -> int | None: + """Return one positive SQLite-safe run ID from an int or canonical decimal.""" + if type(value) is int: + candidate = value + elif type(value) is str: + # Bound conversion before ``int`` so an attacker cannot hand Python an + # arbitrarily large decimal from an otherwise lexically valid name. + if not re.fullmatch(r"[1-9][0-9]{0,18}", value): + return None + candidate = int(value) + else: + return None + if not 1 <= candidate <= SQLITE_RUN_ID_MAX: + return None + return candidate + + +@dataclass(frozen=True) +class TerminalIdentity: + """Replay authority derived only from a journal's directory and filename.""" + + board: str + task_id: str + run_id: int + state: str + + def __post_init__(self) -> None: + """Reject noncanonical identities before they can reach SQLite.""" + if type(self.run_id) is not int or canonical_run_id(self.run_id) is None: + raise ValueError("terminal run identity is not a positive SQLite int64") + + +@dataclass +class TerminalSnapshot: + """One inode-bound, bounded journal read held open across finalization.""" + + document: dict[str, Any] + file_stat: os.stat_result + descriptor: int + directory_descriptor: int + + def close(self) -> None: + """Release the pinned file and directory descriptors.""" + os.close(self.descriptor) + os.close(self.directory_descriptor) + + +@dataclass +class TerminalRecoverySnapshot: + """One raw directory entry pinned before recovery classifies its payload.""" + + document: dict[str, Any] | None + file_stat: os.stat_result + descriptor: int | None + directory_descriptor: int + prefix: bytes + invalid_reason: str | None + + def close(self) -> None: + """Release the descriptors retained across classification/quarantine.""" + if self.descriptor is not None: + os.close(self.descriptor) + self.descriptor = None + if self.directory_descriptor >= 0: + os.close(self.directory_descriptor) + self.directory_descriptor = -1 + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py new file mode 100644 index 00000000..7d386a8c --- /dev/null +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Concurrent board recovery, claim, and dispatch loop for CLI workers.""" + +from __future__ import annotations + +import concurrent.futures +import os +import sys +import time +from typing import Any + +from cli_lane_board import _external, _record_board_access_error, _task_value +from cli_lane_config import ( + BOARD_CORRUPTION_ERRORS, + DEFAULT_CLAIM_TTL, + EXTERNAL_PREFIX, + RESULT_SCHEMA, + RESULT_SCHEMA_PATH, +) +from cli_lane_execution import execute_claim +from cli_lane_files import atomic_json +from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations +from cli_lane_retention import maybe_gc_lane_artifacts + + +def _board_slug(board: Any) -> str: + if isinstance(board, dict): + return str(board.get("slug") or board.get("id") or "") + return str(getattr(board, "slug", None) or getattr(board, "id", None) or board) + +def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None: + """Open one board without letting localized storage faults stop other lanes.""" + try: + return kanban_db.connect(board=board) + except Exception as error: + _record_board_access_error(board, error) + return None + +def recover_orphans() -> None: + """Return external running tasks to ready after a runner/pod restart.""" + from hermes_cli import kanban_db + + recover_pending_finalizations() + try: + boards = kanban_db.list_boards(include_archived=False) + except Exception as error: + _record_board_access_error("board-registry", error) + return + for raw_board in boards: + board = _board_slug(raw_board) + if not board: + continue + with kanban_db.scoped_current_board(board): + conn = _connect_healthy_board(kanban_db, board) + if conn is None: + continue + try: + for task in kanban_db.list_tasks(conn): + if _external(task) and str(_task_value(task, "status", "")) == "running": + task_id = str(_task_value(task, "id")) + run_id = _task_value(task, "current_run_id", None) + if not isinstance(run_id, int): + continue + if _has_pending_finalization(board, task_id, run_id): + continue + kanban_db.reclaim_task( + conn, + task_id, + reason="direct CLI lane restarted; provider session will resume", + expected_run_id=run_id, + ) + BOARD_CORRUPTION_ERRORS.pop(board, None) + except Exception as error: + _record_board_access_error(board, error) + finally: + conn.close() + +def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: + """Atomically claim external ready tasks across all non-archived boards.""" + from hermes_cli import kanban_db + + claimed: list[tuple[str, str]] = [] + if limit <= 0: + return claimed + for raw_board in kanban_db.list_boards(include_archived=False): + board = _board_slug(raw_board) + if not board: + continue + with kanban_db.scoped_current_board(board): + conn = _connect_healthy_board(kanban_db, board) + if conn is None: + continue + try: + kanban_db.recompute_ready(conn) + tasks = kanban_db.list_tasks(conn) + BOARD_CORRUPTION_ERRORS.pop(board, None) + for task in tasks: + task_id = str(_task_value(task, "id", "")) + assignee = str(_task_value(task, "assignee", "") or "") + if task_id and not assignee and str(_task_value(task, "status", "")) == "ready": + if kanban_db.assign_task(conn, task_id, "cli-auto"): + task = kanban_db.get_task(conn, task_id) + assignee = "cli-auto" + if ( + not task_id + or (board, task_id) in active + or not assignee.startswith(EXTERNAL_PREFIX) + or str(_task_value(task, "status", "")) != "ready" + ): + continue + try: + result = kanban_db.claim_task( + conn, + task_id, + ttl_seconds=DEFAULT_CLAIM_TTL, + claimer="direct-cli-lane", + ) + except Exception: + continue + if result is not None: + claimed.append((board, task_id)) + if len(claimed) >= limit: + return claimed + except Exception as error: + _record_board_access_error(board, error) + finally: + conn.close() + return claimed + +def main() -> int: + """Continuously bridge external Kanban lanes to provider CLIs.""" + RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644) + recover_orphans() + workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8)) + futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + while True: + for future in list(futures): + if future.done(): + try: + future.result() + except Exception as error: + print(f"worker future failed: {error}", file=sys.stderr, flush=True) + del futures[future] + recover_pending_finalizations() + maybe_gc_lane_artifacts() + active = set(futures.values()) + try: + newly_claimed = claim_ready(active, workers - len(futures)) + BOARD_CORRUPTION_ERRORS.pop("board-registry", None) + except Exception as error: + _record_board_access_error("board-registry", error) + newly_claimed = [] + for board, task_id in newly_claimed: + future = pool.submit(execute_claim, board, task_id) + futures[future] = (board, task_id) + time.sleep(5) + return 0 diff --git a/services/hermes/scripts/cli_lane_evidence.py b/services/hermes/scripts/cli_lane_evidence.py new file mode 100644 index 00000000..f261e870 --- /dev/null +++ b/services/hermes/scripts/cli_lane_evidence.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""Immutable terminal evidence creation and identity-safe retirement.""" + +from __future__ import annotations + +import ctypes +import errno +import hashlib +import json +import os +import stat +import uuid +from pathlib import Path +from typing import Any + +from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now +from cli_lane_files import _terminal_evidence_identity, state_path +from cli_lane_records import _load_small_json, _terminal_record_valid + + +def _rename_noreplace( + source: str, + destination: str, + *, + source_dir: int, + destination_dir: int, +) -> None: + """Atomically rename a directory entry without replacing a collision.""" + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None) + if renameat2 is None: + raise OSError(errno.ENOSYS, "renameat2 is unavailable") + renameat2.argtypes = ( + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ) + renameat2.restype = ctypes.c_int + result = renameat2( + source_dir, + ctypes.c_char_p(os.fsencode(source)), + destination_dir, + ctypes.c_char_p(os.fsencode(destination)), + 1, # RENAME_NOREPLACE + ) + if result != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), source, destination) + +def _retire_terminal_entry( + path: Path, + source_stat: os.stat_result, + *, + board_descriptor: int, + quarantine_descriptor: int, + authority_name: str | None = None, +) -> str: + """Retire only the inode previously inspected; preserve any replacement.""" + try: + current = os.stat( + path.name, + dir_fd=board_descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + return "missing" + if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino: + return "replacement" + # A replacement can itself be staged repeatedly while recovery races a + # writer. Keep every generation bound to the original canonical pending + # name instead of hashing an intermediate .retire name and hiding it from + # the next recovery pass. + path_digest = hashlib.sha256( + (authority_name or path.name).encode("utf-8") + ).hexdigest()[:16] + source_digest = hashlib.sha256( + f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode("utf-8") + ).hexdigest()[:16] + for sequence in range(32): + staging = f".retire.{path_digest}.{source_digest}.{sequence}" + try: + _rename_noreplace( + path.name, + staging, + source_dir=board_descriptor, + destination_dir=quarantine_descriptor, + ) + except FileExistsError: + continue + break + else: + return "collision" + staged = os.stat( + staging, + dir_fd=quarantine_descriptor, + follow_symlinks=False, + ) + if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino: + os.unlink(staging, dir_fd=quarantine_descriptor) + os.fsync(quarantine_descriptor) + os.fsync(board_descriptor) + return "retired" + # The path changed after the pre-check but before the rename. Put the + # replacement back without overwriting an even newer entry. + try: + _rename_noreplace( + staging, + path.name, + source_dir=quarantine_descriptor, + destination_dir=board_descriptor, + ) + except FileExistsError: + return "replacement-staged" + os.fsync(quarantine_descriptor) + os.fsync(board_descriptor) + return "replacement" + +def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: + """Durably create one immutable JSON artifact without replacing a peer.""" + payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8") + if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096: + raise ValueError("terminal evidence exceeds the bounded artifact limit") + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(path.parent, 0o700, follow_symlinks=False) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + create_flags |= getattr(os, "O_NOFOLLOW", 0) + directory = os.open(path.parent, directory_flags) + temporary = f".{path.name}.{uuid.uuid4().hex}.tmp" + temporary_stat = None + descriptor = None + try: + descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory) + temporary_stat = os.fstat(descriptor) + with os.fdopen(descriptor, "wb") as stream: + descriptor = None + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + written = os.fstat(stream.fileno()) + if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1: + raise OSError("terminal evidence is not private and singly linked") + try: + _rename_noreplace( + temporary, + path.name, + source_dir=directory, + destination_dir=directory, + ) + except FileExistsError: + return False + os.fsync(directory) + return True + finally: + if descriptor is not None: + os.close(descriptor) + if temporary_stat is not None: + try: + _retire_terminal_entry( + Path(temporary), + temporary_stat, + board_descriptor=directory, + quarantine_descriptor=directory, + ) + except OSError: + pass + os.close(directory) + +def _terminal_document_digest(document: dict[str, Any]) -> str: + """Return the stable identity for one exact accepted result document.""" + immutable = { + key: document.get(key) + for key in ( + "board", + "task_id", + "expected_run_id", + "result", + "summary", + "metadata", + ) + } + canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32] + +def _terminal_evidence_path( + identity: TerminalIdentity, + state: str, + document: dict[str, Any], +) -> Path: + """Return a deterministic immutable prepared or conflict evidence path.""" + if state not in {"prepared", "conflict"}: + raise ValueError(f"invalid terminal evidence state: {state}") + base = state_path(identity.board, identity.task_id) + digest = _terminal_document_digest(document) + return base.with_name( + f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json" + ) + +def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool: + """Compare the immutable result-bearing fields of terminal evidence.""" + keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata") + return all(left.get(key) == right.get(key) for key in keys) + +def _terminal_evidence_valid( + document: dict[str, Any], + identity: TerminalIdentity, + state: str, +) -> bool: + """Validate semantic terminal content plus its immutable evidence state.""" + if document.get("kanban_state") != state: + return False + semantic = dict(document) + semantic_identity = identity + if state == "conflict": + semantic["kanban_state"] = "prepared" + semantic_identity = TerminalIdentity( + identity.board, + identity.task_id, + identity.run_id, + "prepared", + ) + return _terminal_record_valid(semantic, semantic_identity) + +def _persist_prepared_evidence( + identity: TerminalIdentity, + document: dict[str, Any], +) -> Path: + """Fsync an immutable accepted result before entering the DB boundary.""" + prepared = dict(document) + prepared["kanban_state"] = "prepared" + prepared["prepared_at"] = utc_now() + path = _terminal_evidence_path(identity, "prepared", document) + if _write_json_noreplace(path, prepared): + return path + if _terminal_evidence_identity(path) != TerminalIdentity( + identity.board, identity.task_id, identity.run_id, "prepared" + ): + raise OSError("prepared terminal evidence identity is invalid") + existing = _load_small_json(path) + prepared_identity = TerminalIdentity( + identity.board, + identity.task_id, + identity.run_id, + "prepared", + ) + if not _terminal_evidence_valid( + existing, + prepared_identity, + "prepared", + ) or not _same_terminal_document(existing, prepared): + raise OSError("prepared terminal evidence path contains a conflicting result") + return path + +def _persist_conflict_evidence( + identity: TerminalIdentity, + document: dict[str, Any], + reason: str, +) -> Path: + """Preserve a losing valid result in full under deterministic retention.""" + conflict = dict(document) + conflict["kanban_state"] = "conflict" + conflict["conflict_reason"] = reason + conflict["conflicted_at"] = utc_now() + path = _terminal_evidence_path(identity, "conflict", document) + if _write_json_noreplace(path, conflict): + return path + if _terminal_evidence_identity(path) != TerminalIdentity( + identity.board, identity.task_id, identity.run_id, "conflict" + ): + raise OSError("terminal conflict evidence identity is invalid") + existing = _load_small_json(path) + conflict_identity = TerminalIdentity( + identity.board, + identity.task_id, + identity.run_id, + "conflict", + ) + if not _terminal_evidence_valid( + existing, + conflict_identity, + "conflict", + ) or not _same_terminal_document(existing, conflict): + raise OSError("terminal conflict evidence path contains a different result") + return path diff --git a/services/hermes/scripts/cli_lane_execution.py b/services/hermes/scripts/cli_lane_execution.py new file mode 100644 index 00000000..3b24b468 --- /dev/null +++ b/services/hermes/scripts/cli_lane_execution.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""Single-claim provider execution and durable Kanban result handoff.""" + +from __future__ import annotations + +import json +import sqlite3 +import time +from pathlib import Path + +import cli_lane_goal +from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value +from cli_lane_config import ( + DEFAULT_MAX_RUNTIME, + TerminalFinalizationPending, +) +from cli_lane_files import ( + _terminal_identity, + _terminal_path, + atomic_json, + load_json, + state_path, +) +from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run +from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts +from cli_lane_provider import run_provider +from cli_lane_records import _persist_candidate, _write_terminal_record +from cli_lane_recovery import _has_pending_finalization +from cli_lane_routing import fresh_unavailable_provider, select_route + + +def execute_claim(board: str, task_id: str) -> None: + """Execute one already-claimed task and commit its outcome to Kanban.""" + from hermes_cli import kanban_db + + run_id = None + with kanban_db.scoped_current_board(board): + conn = kanban_db.connect(board=board) + try: + task = kanban_db.get_task(conn, task_id) + if task is None: + return + run_id = _task_value(task, "current_run_id", None) + assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto") + state_file = state_path(board, task_id) + state = load_json(state_file) + state.update( + { + "board": board, + "task_id": task_id, + "run_id": run_id, + "assignee": assignee, + } + ) + atomic_json(state_file, state) + log_path = Path(kanban_db.worker_log_path(task_id, board=board)) + workspace = _resolve_workspace(kanban_db, conn, task, board) + kanban_db.set_workspace_path(conn, task_id, str(workspace)) + context = _task_context(kanban_db, conn, task_id) + except Exception as error: + failure_reason = ( + f"Direct CLI lane preparation failed: {type(error).__name__}: {error}" + ) + conn.close() + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.block_task( + fresh, + task_id, + reason=failure_reason, + kind="capability", + expected_run_id=run_id, + ), + ) + return + finally: + conn.close() + + def heartbeat(note: str) -> bool: + try: + return bool( + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.heartbeat_worker( + fresh, + task_id, + note=note, + expected_run_id=run_id, + ), + ) + ) + except (OSError, sqlite3.Error): + # The claim TTL is deliberately long. Keep the expensive + # provider process alive during a transient volume stall and + # retry on the next heartbeat instead of losing its work. + return True + + def comment(body: str) -> None: + try: + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.add_comment( + fresh, + task_id, + "cli-lane-runner", + body, + ), + ) + except (OSError, sqlite3.Error): + # Route state is also written to the durable lane-state file; + # a later heartbeat or terminal result remains authoritative. + pass + + try: + previous_route = state.get("current_route") + excluded_provider = ( + fresh_unavailable_provider() if assignee == "cli-auto" else None + ) + unavailable_provider = excluded_provider + route = select_route( + context, + assignee, + exclude_provider=excluded_provider, + exclude_reason="is unavailable according to fresh native health" + if excluded_provider + else None, + ) + if excluded_provider: + comment( + f"Provider health guard excluded {excluded_provider} before automatic routing.", + ) + comment( + f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}", + ) + resume_handoff = "" + if ( + isinstance(previous_route, dict) + and previous_route.get("provider") + and previous_route.get("provider") != route.provider + ): + try: + previous_output = log_path.read_text(encoding="utf-8")[-10000:] + except OSError: + previous_output = "Previous provider log was unavailable after restart." + resume_handoff = git_handoff(workspace, previous_output) + comment( + f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.", + ) + prompt = build_prompt(context, workspace, resume_handoff) + max_runtime = int( + _task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME + ) + goal_mode = bool(_task_value(task, "goal_mode", False)) + goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1)) + goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1) + deadline = time.monotonic() + max_runtime + while True: + state["goal_turn"] = goal_turn + atomic_json(state_file, state) + remaining = max(1, int(deadline - time.monotonic())) + result = run_provider( + route, + prompt, + workspace, + state, + state_file, + log_path, + heartbeat, + remaining, + ) + candidate_file = None + if result.structured: + candidate_file = _persist_candidate( + state, + state_file, + dict(result.structured), + route=route, + returncode=result.returncode, + goal_turn=goal_turn, + ) + if result.capacity_failure: + unavailable_provider = route.provider + retry_context = ( + context + + "\n\nRouting boundary: the first provider failed from capacity/authentication. " + + "Select the alternate hosted provider at an appropriate effort." + ) + alternate = "claude" if route.provider == "codex" else "codex" + fallback = select_route( + retry_context, + f"cli-{alternate}-{route.effort}", + ) + comment( + f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.", + ) + result = run_provider( + fallback, + build_prompt( + context, + workspace, + git_handoff(workspace, result.output), + ), + workspace, + state, + state_file, + log_path, + heartbeat, + max(1, int(deadline - time.monotonic())), + ) + route = fallback + if result.structured: + candidate_file = _persist_candidate( + state, + state_file, + dict(result.structured), + route=route, + returncode=result.returncode, + goal_turn=goal_turn, + ) + structured = result.structured + if structured: + structured = dict(structured) + structured["artifacts"] = workspace_artifacts( + workspace, + structured.get("artifacts"), + ) + metadata = { + "executor": "direct-cli-lane", + "provider": route.provider, + "model": route.model, + "effort": route.effort, + "classifier": route.classifier, + "state_file": str(state_file), + "codex_thread_id": state.get("codex_thread_id"), + "claude_session_id": state.get("claude_session_id"), + "goal_mode": goal_mode, + "goal_turn": goal_turn, + } + if candidate_file is not None: + metadata["candidate_file"] = str(candidate_file) + if structured: + for key in ( + "changed_files", + "tests_run", + "artifacts", + "findings", + "blockers", + ): + value = structured.get(key) + metadata[key] = value if isinstance(value, list) else [] + + completion_problem = None + if structured and result.returncode == 0: + completion_problem = cli_lane_goal.unfinished_result_reason(structured) + if ( + structured.get("status") == "completed" + and completion_problem is None + and goal_mode + ): + heartbeat("local goal-completion judge active") + rejection_history = state.get("goal_rejections", []) + if not isinstance(rejection_history, list): + rejection_history = [] + judge_context = context + if goal_turn > 1 or rejection_history: + judge_context += ( + "\n\nAuthoritative Hermes goal-controller evidence: " + f"current turn {goal_turn}/{goal_max_turns}; prior rejected " + f"reports: {json.dumps(rejection_history[-5:])}." + ) + accepted, judge_reason = cli_lane_goal.judge_goal_completion( + judge_context, + structured, + ) + metadata["goal_judge_reason"] = judge_reason + if not accepted: + completion_problem = f"local goal judge requested continuation: {judge_reason}" + + if ( + structured + and structured.get("status") == "completed" + and result.returncode == 0 + and completion_problem is None + ): + terminal_file = _terminal_path(state_file, run_id, "pending") + metadata["terminal_record"] = str( + _terminal_path(state_file, run_id, "committed") + ) + try: + terminal_file, terminal_record = _write_terminal_record( + state_file, + board=board, + task_id=task_id, + run_id=run_id, + structured=structured, + summary=str(structured.get("summary") or "Completed"), + metadata=metadata, + ) + outcome = _finalize_terminal_record( + kanban_db, + terminal_file, + terminal_record, + ) + except Exception as error: + identity = _terminal_identity(terminal_file) + replayable = bool( + identity is not None + and _has_pending_finalization( + identity.board, + identity.task_id, + identity.run_id, + ) + ) + if not replayable: + _recover_exact_run( + kanban_db, + identity, + f"persistence raised {type(error).__name__}", + ) + raise TerminalFinalizationPending( + "accepted worker result remains outside the generic block path; " + f"terminal replayable={replayable}; persistence/finalization " + f"raised {type(error).__name__}" + ) from error + if outcome != "committed": + raise TerminalFinalizationPending( + "accepted worker result is durably journaled but " + f"Kanban finalization is {outcome}" + ) + break + + can_continue = ( + goal_mode + and completion_problem is not None + and goal_turn < goal_max_turns + and deadline - time.monotonic() > 30 + ) + if can_continue: + rejection_history = state.get("goal_rejections", []) + if not isinstance(rejection_history, list): + rejection_history = [] + state["goal_rejections"] = [ + *rejection_history[-4:], + completion_problem, + ] + goal_turn += 1 + comment( + f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}", + ) + escalation_context = ( + context + + "\n\nThe previous worker turn missed its completion quality mark: " + + completion_problem + + "\nSelect a route that can finish and verify the remaining work." + ) + excluded = unavailable_provider + if excluded is None and assignee == "cli-auto": + excluded = fresh_unavailable_provider() + next_route = select_route( + escalation_context, + assignee, + exclude_provider=excluded, + exclude_reason="is unavailable according to fresh native health" + if excluded + else None, + ) + comment( + f"Goal route {goal_turn}/{goal_max_turns}: " + f"{next_route.provider}/{next_route.model} at {next_route.effort}; " + f"classifier={next_route.classifier}; {next_route.reason}", + ) + handoff = ( + git_handoff(workspace, result.output) + if next_route.provider != route.provider + else "" + ) + route = next_route + prompt = build_prompt( + context, + workspace, + handoff + + "\nGoal-loop continuation: the previous final report was rejected because " + + completion_problem + + ". Reinspect live state, finish the outstanding work, and return new final evidence.", + ) + continue + + reason = completion_problem + if reason is None and structured: + blockers = structured.get("blockers", []) + reason = "; ".join(str(item) for item in blockers) + reason = reason or str(structured.get("summary") or "") + if reason is None: + reason = result.output[-4000:] + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.block_task( + fresh, + task_id, + reason=reason + or f"{route.provider} worker failed with exit {result.returncode}", + kind="transient" if result.capacity_failure else "capability", + expected_run_id=run_id, + ), + ) + break + except TerminalFinalizationPending as error: + comment(str(error)) + except Exception as error: + failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}" + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.block_task( + fresh, + task_id, + reason=failure_reason, + kind="capability", + expected_run_id=run_id, + ), + ) diff --git a/services/hermes/scripts/cli_lane_files.py b/services/hermes/scripts/cli_lane_files.py new file mode 100644 index 00000000..9d160da0 --- /dev/null +++ b/services/hermes/scripts/cli_lane_files.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Durable path and filesystem primitives for Hermes CLI lane journals.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import uuid +from pathlib import Path +from typing import Any + +from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id + + +def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: + """Durably replace a small state document without following temp symlinks.""" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(path.parent, 0o700, follow_symlinks=False) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = None + try: + descriptor = os.open(temporary, flags, mode) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + descriptor = None + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, mode, follow_symlinks=False) + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + if descriptor is not None: + os.close(descriptor) + try: + temporary.unlink() + except FileNotFoundError: + pass + +def _fsync_directory(directory: Path) -> None: + """Persist directory-entry changes after an atomic rename or quarantine.""" + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(directory, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + +def load_json(path: Path) -> dict[str, Any]: + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "r", encoding="utf-8") as stream: + value = json.load(stream) + except (OSError, UnicodeError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + +def state_path(board: str, task_id: str) -> Path: + safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board) + safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id) + return STATE_ROOT / safe_board / f"{safe_task}.json" + +def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path: + """Return a unique provider-result path; completed turns are never reused.""" + safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) + return state_file.with_name( + f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json" + ) + +def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path: + """Return the durable path for one exact structured worker response.""" + safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) + return state_file.with_name( + f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json" + ) + +def _terminal_path( + state_file: Path, + run_id: Any, + state: str = "pending", +) -> Path: + """Return the identity-bound replay path for an accepted terminal response.""" + if state not in {"pending", "committed"}: + raise ValueError(f"invalid terminal journal state: {state}") + canonical = canonical_run_id(run_id) + if type(run_id) is not int or canonical is None: + raise ValueError("terminal run ID must be a positive SQLite int64") + return state_file.with_name( + f"{state_file.stem}.run-{canonical}.terminal.{state}.json" + ) + +_TERMINAL_NAME = re.compile( + r"^(?P[a-zA-Z0-9_.-]+)\.run-(?P[0-9]+)\." + r"terminal\.(?Ppending|committed)\.json$" +) +_TERMINAL_EVIDENCE_NAME = re.compile( + r"^(?P[a-zA-Z0-9_.-]+)\.run-(?P[0-9]+)\." + r"terminal\.(?Pprepared|conflict)-(?P[a-f0-9]{32})\.json$" +) +_RETIRE_NAME = re.compile( + r"^\.retire\.(?P[a-f0-9]{16})\." + r"(?P[a-f0-9]{16})\.(?P[0-9]+)$" +) +_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$") +_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$") + + +def _terminal_identity(path: Path) -> TerminalIdentity | None: + """Parse replay authority lexically before opening a journal or board.""" + try: + relative = path.relative_to(STATE_ROOT) + except ValueError: + return None + if len(relative.parts) != 2: + return None + board, filename = relative.parts + match = _TERMINAL_NAME.fullmatch(filename) + run_id = canonical_run_id(match.group("run")) if match else None + try: + board_stat = path.parent.stat(follow_symlinks=False) + except OSError: + return None + if ( + not match + or run_id is None + or not stat.S_ISDIR(board_stat.st_mode) + or not _SAFE_BOARD.fullmatch(board) + or board in {".", ".."} + or match.group("task") in {".", ".."} + ): + return None + return TerminalIdentity( + board=board, + task_id=match.group("task"), + run_id=run_id, + state=match.group("state"), + ) + +def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None: + """Parse immutable prepared/conflict evidence authority from its path.""" + try: + relative = path.relative_to(STATE_ROOT) + except ValueError: + return None + if len(relative.parts) != 2: + return None + board, filename = relative.parts + match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename) + run_id = canonical_run_id(match.group("run")) if match else None + try: + board_stat = path.parent.stat(follow_symlinks=False) + except OSError: + return None + if ( + not match + or run_id is None + or not stat.S_ISDIR(board_stat.st_mode) + or not _SAFE_BOARD.fullmatch(board) + or board in {".", ".."} + or match.group("task") in {".", ".."} + ): + return None + return TerminalIdentity( + board=board, + task_id=match.group("task"), + run_id=run_id, + state=match.group("state"), + ) diff --git a/services/hermes/scripts/cli_lane_finalization.py b/services/hermes/scripts/cli_lane_finalization.py new file mode 100644 index 00000000..5599f830 --- /dev/null +++ b/services/hermes/scripts/cli_lane_finalization.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Exact-run database completion and terminal evidence publication.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +from cli_lane_board import _board_call, _external, _record_board_access_error, _task_value +from cli_lane_config import ( + TerminalIdentity, + TerminalRecoverySnapshot, + TerminalSnapshot, + canonical_run_id, + utc_now, +) +from cli_lane_evidence import ( + _persist_conflict_evidence, + _persist_prepared_evidence, + _retire_terminal_entry, + _same_terminal_document, + _write_json_noreplace, +) +from cli_lane_files import _terminal_identity, _terminal_path, state_path +from cli_lane_records import ( + _load_terminal_json, + _open_small_json_snapshot, + _open_terminal_snapshot, + _terminal_record_valid, +) + + +def _recover_exact_run( + kanban_db: Any, + identity: TerminalIdentity | None, + reason: str, +) -> bool: + """Make an exact external run retryable after journal recovery fails.""" + if identity is None or canonical_run_id(identity.run_id) is None: + return False + + def operation(conn: Any) -> bool: + task = kanban_db.get_task(conn, identity.task_id) + if task is None: + return False + if ( + str(_task_value(task, "status", "")) != "running" + or _task_value(task, "current_run_id", None) != identity.run_id + or not _external(task) + ): + return False + return bool( + kanban_db.reclaim_task( + conn, + identity.task_id, + reason=f"terminal journal recovery failed ({reason}); exact run may retry", + expected_run_id=identity.run_id, + ) + ) + + try: + recovered = bool(_board_call(kanban_db, identity.board, operation)) + except Exception as error: + _record_board_access_error(identity.board, error) + return False + if recovered: + print( + f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} " + f"after terminal journal recovery: {reason}", + file=sys.stderr, + flush=True, + ) + return recovered + +def _retire_snapshot( + path: Path, + snapshot: TerminalSnapshot, + *, + authority_name: str | None = None, +) -> str: + """Retire only the directory entry still naming an opened snapshot inode.""" + return _retire_terminal_entry( + path, + snapshot.file_stat, + board_descriptor=snapshot.directory_descriptor, + quarantine_descriptor=snapshot.directory_descriptor, + authority_name=authority_name, + ) + +def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str: + """Keep a committed DB result recoverable across retirement fsync errors.""" + try: + return _retire_snapshot(path, snapshot) + except OSError as error: + print( + f"terminal journal retirement deferred after DB commit: " + f"{type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + return "deferred" + +def _discard_evidence(path: Path) -> str: + """Identity-safely remove one immutable prepared evidence artifact.""" + snapshot = _open_small_json_snapshot(path) + if snapshot is None: + return "missing" + try: + return _retire_snapshot(path, snapshot) + finally: + snapshot.close() + +def _promote_prepared_evidence( + identity: TerminalIdentity, + document: dict[str, Any], + prepared: Path, + *, + discard_prepared: bool = True, +) -> tuple[Path, str]: + """Publish the DB-winning result without overwriting first-writer evidence.""" + committed = _terminal_path( + state_path(identity.board, identity.task_id), + identity.run_id, + "committed", + ) + committed_document = dict(document) + committed_document["kanban_state"] = "committed" + committed_document["committed_at"] = utc_now() + state = "committed" + if not _write_json_noreplace(committed, committed_document): + committed_identity = TerminalIdentity( + identity.board, + identity.task_id, + identity.run_id, + "committed", + ) + existing = _load_terminal_json(committed, committed_identity) + if ( + not _terminal_record_valid(existing, committed_identity) + or existing.get("kanban_state") != "committed" + or not _same_terminal_document(existing, committed_document) + ): + _persist_conflict_evidence( + identity, + document, + "canonical committed evidence already contains a different result", + ) + state = "conflict" + if discard_prepared: + _discard_evidence(prepared) + return committed, state + +def _finalize_document_db( + kanban_db: Any, + identity: TerminalIdentity, + document: dict[str, Any], +) -> str: + """Apply one validated exact-run result to Kanban under DB run guards.""" + if canonical_run_id(identity.run_id) is None: + return "stale" + + def operation(conn: Any) -> str: + task = kanban_db.get_task(conn, identity.task_id) + if task is None: + return "stale" + status = str(_task_value(task, "status", "")) + if status == "done": + return ( + "committed" + if ( + _task_value(task, "completed_run_id", None) == identity.run_id + and str(_task_value(task, "result", "") or "") + == document["result"] + ) + else "stale" + ) + current_run_id = _task_value(task, "current_run_id", None) + completion_guard: dict[str, int] + if status == "running" and current_run_id == identity.run_id: + completion_guard = {"expected_run_id": identity.run_id} + elif status in {"ready", "blocked", "triage"} and current_run_id is None: + # The journal may have survived an older post-persistence error + # path that ended its run. The patched DB verifies atomically that + # this is still the latest ended run before allowing completion. + completion_guard = {"replay_ended_run_id": identity.run_id} + else: + return "stale" + completed = kanban_db.complete_task( + conn, + identity.task_id, + result=document["result"], + summary=document["summary"], + metadata=document["metadata"], + **completion_guard, + ) + if completed: + return "committed" + # A duplicate finalizer can lose the guarded UPDATE to an identical + # first writer. Re-read rather than reporting a false pending state. + latest = kanban_db.get_task(conn, identity.task_id) + latest_status = ( + str(_task_value(latest, "status", "")) if latest is not None else "" + ) + if latest_status == "done": + return ( + "committed" + if ( + _task_value(latest, "completed_run_id", None) == identity.run_id + and str(_task_value(latest, "result", "") or "") + == document["result"] + ) + else "stale" + ) + # A guarded completion can be retried only while this exact live run + # still owns the task. An ended, replaced, or nonexistent run cannot + # become authoritative later and must converge to conflict evidence. + if ( + latest_status == "running" + and _task_value(latest, "current_run_id", None) == identity.run_id + ): + return "pending" + return "stale" + + return str(_board_call(kanban_db, identity.board, operation)) + +def _resolve_pending_after_winner( + path: Path, + identity: TerminalIdentity, + winning_document: dict[str, Any], +) -> bool: + """Retire duplicates and preserve differing replacements as conflicts.""" + for _attempt in range(8): + snapshot = _open_terminal_snapshot(path, identity) + if snapshot is None: + return False + document = snapshot.document + if not _terminal_record_valid(document, identity): + snapshot.close() + return False + differing = not _same_terminal_document(document, winning_document) + try: + if differing: + _persist_conflict_evidence( + identity, + document, + "different valid result lost the exact-run first-writer race", + ) + retirement = _retire_snapshot_after_db(path, snapshot) + finally: + snapshot.close() + if retirement in {"retired", "missing"}: + if differing: + print( + f"preserved terminal result conflict for {identity.board}/" + f"{identity.task_id} run {identity.run_id}", + file=sys.stderr, + flush=True, + ) + return differing + if retirement not in {"replacement", "replacement-staged", "collision"}: + return differing + return False + +def _finalize_terminal_record( + kanban_db: Any, + path: Path, + _record: dict[str, Any] | None = None, + *, + snapshot: TerminalRecoverySnapshot | None = None, +) -> str: + """Commit one inode-pinned exact-run journal, or leave it safely pending.""" + identity = _terminal_identity(path) + if identity is None or identity.state != "pending": + if snapshot is not None: + snapshot.close() + return "invalid" + pinned: TerminalSnapshot | TerminalRecoverySnapshot | None = snapshot + if pinned is None: + pinned = _open_terminal_snapshot(path, identity) + if pinned is None or pinned.document is None: + if pinned is not None: + pinned.close() + return "invalid" + document = pinned.document + try: + if not _terminal_record_valid(document): + return "invalid" + if not _terminal_record_valid(document, identity): + return "foreign" + prepared = _persist_prepared_evidence(identity, document) + outcome = _finalize_document_db(kanban_db, identity, document) + if outcome == "committed": + _committed, evidence_state = _promote_prepared_evidence( + identity, + document, + prepared, + ) + retirement = _retire_snapshot_after_db(path, pinned) + elif outcome == "stale": + _persist_conflict_evidence( + identity, + document, + "valid terminal result no longer matches the authoritative task run", + ) + _discard_evidence(prepared) + retirement = _retire_snapshot_after_db(path, pinned) + outcome = "conflict" + evidence_state = "conflict" + else: + return outcome + finally: + pinned.close() + if outcome == "committed" and retirement not in {"retired", "missing"}: + _resolve_pending_after_winner(path, identity, document) + if evidence_state == "conflict": + print( + f"terminal first-writer conflict recorded for {identity.board}/" + f"{identity.task_id} run {identity.run_id}", + file=sys.stderr, + flush=True, + ) + return outcome diff --git a/services/hermes/scripts/cli_lane_prompt.py b/services/hermes/scripts/cli_lane_prompt.py new file mode 100644 index 00000000..59e36889 --- /dev/null +++ b/services/hermes/scripts/cli_lane_prompt.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Prompt, artifact, and structured-event helpers for CLI workers.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path +from typing import Any + +import cli_lane_goal +from cli_lane_files import atomic_json + + +def build_prompt(context: str, workspace: Path, handoff: str = "") -> str: + """Create a bounded worker contract with an explicit machine-readable result.""" + return f"""You are a durable coding worker managed by Hermes Kanban. + +Work only on this objective and its acceptance criteria: + +{context} + +Workspace: {workspace} +{handoff} + +Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials. + +Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly. +""" + +def git_handoff(workspace: Path, prior_output: str) -> str: + """Build an explicit cross-provider handoff without transferring hidden state.""" + def read_git(*args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(workspace), *args], + text=True, + capture_output=True, + timeout=30, + check=False, + ) + return (completed.stdout or completed.stderr).strip()[-12000:] + + return ( + "\nCross-provider handoff from a failed or exhausted worker:\n" + f"Git status:\n{read_git('status', '--short', '--branch')}\n\n" + f"Current diff summary:\n{read_git('diff', '--stat')}\n\n" + f"Prior worker tail:\n{prior_output[-10000:]}\n" + "Reinspect the workspace and verify all inherited claims before continuing." + ) + +def workspace_artifacts(workspace: Path, values: Any) -> list[str]: + """Return existing regular artifacts contained by the task worktree.""" + if not isinstance(values, list): + return [] + root = workspace.resolve() + artifacts: list[str] = [] + for value in values: + if not isinstance(value, str) or not value.strip(): + continue + candidate = Path(value.strip()).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (OSError, RuntimeError, ValueError): + continue + if resolved.is_file(): + artifacts.append(str(resolved)) + return list(dict.fromkeys(artifacts)) + +def _extract_json(value: Any) -> dict[str, Any] | None: + if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES: + return value + if not isinstance(value, str): + return None + candidates = [value] + match = re.search(r"\{.*\}", value, re.DOTALL) + if match: + candidates.append(match.group(0)) + for candidate in candidates: + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES: + return parsed + return None + +def _event_payload(provider: str, line: str, state: dict[str, Any], state_file: Path) -> dict[str, Any] | None: + """Persist provider session identifiers before interpreting final output.""" + try: + event = json.loads(line) + except json.JSONDecodeError: + return None + if not isinstance(event, dict): + return None + if provider == "codex" and event.get("type") == "thread.started": + thread_id = event.get("thread_id") or event.get("thread", {}).get("id") + if thread_id: + state["codex_thread_id"] = str(thread_id) + atomic_json(state_file, state) + if provider == "claude" and event.get("session_id"): + state["claude_session_id"] = str(event["session_id"]) + atomic_json(state_file, state) + for key in ("structured_output", "result", "output", "text"): + parsed = _extract_json(event.get(key)) + if parsed: + return parsed + item = event.get("item") + if isinstance(item, dict): + for key in ("text", "content"): + parsed = _extract_json(item.get(key)) + if parsed: + return parsed + return None diff --git a/services/hermes/scripts/cli_lane_provider.py b/services/hermes/scripts/cli_lane_provider.py new file mode 100644 index 00000000..53a05bba --- /dev/null +++ b/services/hermes/scripts/cli_lane_provider.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""Native Codex and Claude process lifecycle for Hermes CLI workers.""" + +from __future__ import annotations + +import json +import os +import selectors +import signal +import subprocess +import time +import uuid +from dataclasses import asdict +from pathlib import Path +from typing import Any, Callable + +import cli_lane_goal +from cli_lane_config import ( + CAPACITY_PATTERN, + CLAUDE_BIN, + CLAUDE_SESSION_COLLISION, + CLAUDE_SETTINGS, + CODEX_BIN, + DATA_ROOT, + HEARTBEAT_SECONDS, + NO_CLAUDE_SESSION, + NO_CODEX_THREAD, + ProcessResult, + RESULT_SCHEMA, + RESULT_SCHEMA_PATH, + Route, + utc_now, +) +from cli_lane_files import _result_path, atomic_json, load_json +from cli_lane_prompt import _event_payload + + +def stream_process( + command: list[str], + *, + provider: str, + cwd: Path, + env: dict[str, str], + log_path: Path, + state: dict[str, Any], + state_file: Path, + heartbeat: Callable[[str], bool], + max_runtime: int, +) -> ProcessResult: + """Stream JSONL to Kanban logs while maintaining the authoritative lease.""" + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + ) + assert process.stdout is not None + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + started = time.monotonic() + last_heartbeat = 0.0 + known_descendants: dict[int, tuple[int, int]] = {} + lines: list[str] = [] + structured: dict[str, Any] | None = None + log_path.parent.mkdir(parents=True, exist_ok=True) + forced_failure = "" + with log_path.open("a", encoding="utf-8") as log: + log.write(f"\n[{utc_now()}] starting {provider} worker\n") + log.flush() + while process.poll() is None: + known_descendants.update(_descendant_processes(process.pid)) + now = time.monotonic() + if now - started > max_runtime: + _terminate_worker_process(process, known_descendants) + forced_failure = "worker exceeded its maximum runtime" + lines.append(forced_failure + "\n") + break + if now - last_heartbeat >= HEARTBEAT_SECONDS: + if not heartbeat(f"{provider} worker active for {round(now - started)}s"): + _terminate_worker_process(process, known_descendants) + forced_failure = "Kanban lease was lost; provider process terminated" + lines.append(forced_failure + "\n") + break + last_heartbeat = now + for key, _ in selector.select(timeout=1.0): + line = key.fileobj.readline() + if not line: + continue + lines.append(line) + if len(lines) > 4000: + lines = lines[-4000:] + log.write(line) + log.flush() + parsed = _event_payload(provider, line, state, state_file) + structured = parsed or structured + _terminate_worker_process(process, known_descendants) + remainder = process.stdout.read() + if remainder: + lines.append(remainder) + log.write(remainder) + for line in remainder.splitlines(): + parsed = _event_payload(provider, line, state, state_file) + structured = parsed or structured + if forced_failure: + log.write(forced_failure + "\n") + log.write(f"\n[{utc_now()}] {provider} exit={process.returncode}\n") + selector.close() + output = "".join(lines)[-100000:] + returncode = int(process.returncode if process.returncode is not None else 1) + return ProcessResult( + returncode=returncode, + output=output, + structured=structured, + capacity_failure=returncode != 0 and bool(CAPACITY_PATTERN.search(output)), + ) + +def _process_record(pid: int) -> tuple[int, int, int] | None: + """Return one Linux process's parent, group, and start-time identity.""" + try: + raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") + fields = raw[raw.rfind(")") + 2 :].split() + return int(fields[1]), int(fields[2]), int(fields[19]) + except (IndexError, OSError, ValueError): + return None + +def _descendant_processes(root_pid: int) -> dict[int, tuple[int, int]]: + """Snapshot descendants as pid -> (process group, start-time identity).""" + records: dict[int, tuple[int, int, int]] = {} + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + record = _process_record(pid) + if record is not None: + records[pid] = record + family = {root_pid} + changed = True + while changed: + changed = False + for pid, (parent, _, _) in records.items(): + if pid not in family and parent in family: + family.add(pid) + changed = True + return { + pid: (records[pid][1], records[pid][2]) + for pid in family + if pid != root_pid and pid in records + } + +def _process_identity_matches(pid: int, start_time: int) -> bool: + record = _process_record(pid) + return record is not None and record[2] == start_time + +def _signal_worker_tree( + root_pid: int, + descendants: dict[int, tuple[int, int]], + sig: signal.Signals, +) -> None: + """Signal captured descendants across terminal-created process groups.""" + groups = {root_pid} + for pid, (process_group, start_time) in descendants.items(): + if _process_identity_matches(pid, start_time): + groups.add(process_group) + for process_group in groups: + try: + os.killpg(process_group, sig) + except ProcessLookupError: + pass + for pid, (_, start_time) in descendants.items(): + if not _process_identity_matches(pid, start_time): + continue + try: + os.kill(pid, sig) + except ProcessLookupError: + pass + +def _terminate_worker_process( + process: subprocess.Popen[str], + known_descendants: dict[int, tuple[int, int]] | None = None, +) -> None: + """Reap a worker and terminal descendants that created their own groups.""" + descendants = dict(known_descendants or {}) + descendants.update(_descendant_processes(process.pid)) + if process.poll() is None: + _signal_worker_tree(process.pid, descendants, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + + # Provider terminal tools create their own sessions, so killing only the + # native CLI's process group leaves test/build subprocesses orphaned. The + # identity check prevents a reused PID from targeting an unrelated worker. + _signal_worker_tree(process.pid, descendants, signal.SIGKILL) + if process.poll() is None: + process.wait(timeout=10) + +def _base_env() -> dict[str, str]: + """Build a worker environment while preserving Vault-backed CLI homes.""" + env = os.environ.copy() + env.update( + { + "HOME": str(DATA_ROOT / "home"), + "CODEX_HOME": os.environ.get("CODEX_HOME", "/runtime-access/codex"), + "CLAUDE_CONFIG_DIR": os.environ.get( + "CLAUDE_CONFIG_DIR", "/runtime-access/claude" + ), + "GIT_TERMINAL_PROMPT": "0", + } + ) + env["PATH"] = f"{DATA_ROOT / 'tools/bin'}:/opt/coordinator:" + env.get("PATH", "") + return env + +def _codex_command(route: Route, prompt: str, workspace: Path, state: dict[str, Any], result_file: Path) -> list[str]: + base = [ + str(CODEX_BIN), + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "exec", + ] + thread_id = str(state.get("codex_thread_id") or "") + common = [ + "--json", + "-m", + route.model, + "-c", + f'model_reasoning_effort="{route.effort}"', + "--output-schema", + str(RESULT_SCHEMA_PATH), + "-o", + str(result_file), + ] + if thread_id: + return [*base, "resume", *common, thread_id, prompt] + return [ + *base, + *common, + "-C", + str(workspace), + prompt, + ] + +def _claude_command(route: Route, prompt: str, state: dict[str, Any], resume: bool) -> list[str]: + session_id = str(state["claude_session_id"]) + session = ["--resume", session_id] if resume else ["--session-id", session_id] + denied = [ + "Bash(git push --force *)", + "Bash(git reset --hard *)", + "Bash(git clean -f *)", + ] + return [ + str(CLAUDE_BIN), + "--dangerously-skip-permissions", + "--autocompact", + "auto", + "--settings", + str(CLAUDE_SETTINGS), + "--disallowedTools", + *denied, + "--model", + route.model, + "--effort", + route.effort, + "--output-format", + "stream-json", + "--verbose", + "--json-schema", + json.dumps(RESULT_SCHEMA, separators=(",", ":")), + *session, + "-p", + prompt, + ] + +def run_provider( + route: Route, + prompt: str, + workspace: Path, + state: dict[str, Any], + state_file: Path, + log_path: Path, + heartbeat: Callable[[str], bool], + max_runtime: int, +) -> ProcessResult: + """Run or resume one provider session using its pinned structured CLI.""" + state.setdefault("attempts", []).append({"route": asdict(route), "started_at": utc_now()}) + if route.provider == "claude" and not state.get("claude_session_id"): + # The reservation is durable before Claude starts, closing the crash gap. + state["claude_session_id"] = str(uuid.uuid4()) + state["current_route"] = asdict(route) + state["updated_at"] = utc_now() + env = _base_env() + if route.provider == "codex": + result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1 + result_file = _result_path(state_file, state.get("run_id"), result_sequence) + while result_file.exists(): + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + result_file.parent.mkdir(parents=True, exist_ok=True) + result_file.touch(mode=0o600, exist_ok=False) + state["result_sequence"] = result_sequence + state["current_result_file"] = str(result_file) + atomic_json(state_file, state) + command = _codex_command(route, prompt, workspace, state, result_file) + result = stream_process( + command, + provider="codex", + cwd=workspace, + env=env, + log_path=log_path, + state=state, + state_file=state_file, + heartbeat=heartbeat, + max_runtime=max_runtime, + ) + if ( + state.get("codex_thread_id") + and result.returncode != 0 + and NO_CODEX_THREAD in result.output.lower() + ): + state.pop("codex_thread_id", None) + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + while result_file.exists(): + result_sequence += 1 + result_file = _result_path( + state_file, + state.get("run_id"), + result_sequence, + ) + result_file.touch(mode=0o600, exist_ok=False) + state["result_sequence"] = result_sequence + state["current_result_file"] = str(result_file) + atomic_json(state_file, state) + result = stream_process( + _codex_command(route, prompt, workspace, state, result_file), + provider="codex", + cwd=workspace, + env=env, + log_path=log_path, + state=state, + state_file=state_file, + heartbeat=heartbeat, + max_runtime=max_runtime, + ) + file_result = load_json(result_file) + if file_result.get("status") in cli_lane_goal.RESULT_STATUSES: + result.structured = file_result + try: + result_file.chmod(0o600) + except OSError: + pass + return result + + atomic_json(state_file, state) + resume = bool(state.get("claude_started")) + result = stream_process( + _claude_command(route, prompt, state, resume), + provider="claude", + cwd=workspace, + env=env, + log_path=log_path, + state=state, + state_file=state_file, + heartbeat=heartbeat, + max_runtime=max_runtime, + ) + if resume and result.returncode == 1 and NO_CLAUDE_SESSION in result.output: + result = stream_process( + _claude_command(route, prompt, state, False), + provider="claude", + cwd=workspace, + env=env, + log_path=log_path, + state=state, + state_file=state_file, + heartbeat=heartbeat, + max_runtime=max_runtime, + ) + elif not resume and result.returncode == 1 and CLAUDE_SESSION_COLLISION in result.output: + result = stream_process( + _claude_command(route, prompt, state, True), + provider="claude", + cwd=workspace, + env=env, + log_path=log_path, + state=state, + state_file=state_file, + heartbeat=heartbeat, + max_runtime=max_runtime, + ) + if result.returncode == 0 or NO_CLAUDE_SESSION not in result.output: + state["claude_started"] = True + atomic_json(state_file, state) + return result diff --git a/services/hermes/scripts/cli_lane_quarantine.py b/services/hermes/scripts/cli_lane_quarantine.py new file mode 100644 index 00000000..21dfddb1 --- /dev/null +++ b/services/hermes/scripts/cli_lane_quarantine.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Bounded diagnostic quarantine for malformed terminal journals.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import sys +from pathlib import Path + +from cli_lane_config import QUARANTINE_HASH_BYTES, STATE_ROOT, TerminalIdentity, TerminalRecoverySnapshot +from cli_lane_evidence import _retire_terminal_entry +from cli_lane_records import _read_bounded + + +def _quarantine_terminal( + path: Path, + identity: TerminalIdentity | None, + reason: str, + *, + snapshot: TerminalRecoverySnapshot | None = None, +) -> Path: + """Record bounded metadata, then retire only the inspected journal inode.""" + board_dir = path.parent + try: + board_stat = board_dir.stat(follow_symlinks=False) + except OSError: + board_stat = None + if board_stat is None or not stat.S_ISDIR(board_stat.st_mode): + print( + f"refused terminal journal path through unsafe board directory: {path}", + file=sys.stderr, + flush=True, + ) + return path + board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + board_flags |= getattr(os, "O_NOFOLLOW", 0) + file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + create_flags |= getattr(os, "O_NOFOLLOW", 0) + board_descriptor = None + quarantine_descriptor = None + source_descriptor = None + source_stat = None + destination = path + retirement = "not-attempted" + owns_source = snapshot is None + try: + relative = path.relative_to(STATE_ROOT) + except ValueError: + relative = Path() + already_quarantined = ( + len(relative.parts) == 3 and relative.parts[1] == "quarantine" + ) + try: + if snapshot is None: + board_descriptor = os.open(board_dir, board_flags) + source_stat = os.stat( + path.name, + dir_fd=board_descriptor, + follow_symlinks=False, + ) + else: + board_descriptor = snapshot.directory_descriptor + source_descriptor = snapshot.descriptor + source_stat = snapshot.file_stat + pinned_board = os.fstat(board_descriptor) + if ( + pinned_board.st_dev != board_stat.st_dev + or pinned_board.st_ino != board_stat.st_ino + ): + raise OSError("terminal recovery board identity changed") + safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1 + digest = None + hashed_bytes = 0 + hash_complete = False + if safe_source and snapshot is None: + source_descriptor = os.open( + path.name, + file_flags, + dir_fd=board_descriptor, + ) + opened_stat = os.fstat(source_descriptor) + if ( + opened_stat.st_dev != source_stat.st_dev + or opened_stat.st_ino != source_stat.st_ino + or opened_stat.st_nlink != 1 + ): + raise OSError("terminal journal identity changed while opening") + # Keep this descriptor open through retirement. The staging entry + # must match the exact fstat-verified inode, not merely a path stat + # captured before a concurrent replacement. + source_stat = opened_stat + prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES) + hashed_bytes = len(prefix) + hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES + digest = hashlib.sha256(prefix).hexdigest() + elif safe_source and source_descriptor is not None: + prefix = snapshot.prefix + hashed_bytes = len(prefix) + hash_complete = ( + source_stat.st_size <= QUARANTINE_HASH_BYTES + and hashed_bytes == source_stat.st_size + ) + digest = hashlib.sha256(prefix).hexdigest() + source_kind = ( + "regular" + if safe_source + else "hardlink" + if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1 + else "non-regular" + ) + diagnostic = { + "hash_complete": hash_complete, + "hashed_bytes": hashed_bytes, + "original_name": path.name, + "reason": reason, + "sha256": digest, + "size": source_stat.st_size, + "source_kind": source_kind, + } + payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8") + fingerprint = ( + (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16] + ) + safe_reason = ( + re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" + ) + if already_quarantined: + # A staged artifact already lives in the board's one bounded + # quarantine directory. Reuse it so adversarial replacements can + # never drive recursive quarantine/quarantine/... authority. + quarantine_descriptor = os.dup(board_descriptor) + quarantine_dir = board_dir + else: + try: + os.mkdir("quarantine", 0o700, dir_fd=board_descriptor) + except FileExistsError: + pass + quarantine_descriptor = os.open( + "quarantine", + board_flags, + dir_fd=board_descriptor, + ) + quarantine_dir = board_dir / "quarantine" + for sequence in range(32): + name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine" + try: + destination_descriptor = os.open( + name, + create_flags, + 0o600, + dir_fd=quarantine_descriptor, + ) + except FileExistsError: + continue + try: + with os.fdopen(destination_descriptor, "wb") as target: + target.write(payload) + target.flush() + os.fsync(target.fileno()) + target_stat = os.fstat(target.fileno()) + if ( + stat.S_IMODE(target_stat.st_mode) != 0o600 + or target_stat.st_nlink != 1 + ): + raise OSError("quarantine destination is not private and singly linked") + except Exception: + os.unlink(name, dir_fd=quarantine_descriptor) + raise + destination = quarantine_dir / name + break + else: + raise OSError("could not reserve a unique quarantine destination") + os.fsync(quarantine_descriptor) + retirement = _retire_terminal_entry( + path, + source_stat, + board_descriptor=board_descriptor, + quarantine_descriptor=quarantine_descriptor, + ) + except OSError as error: + # A bad quarantine target must not make an attacker-controlled pending + # path eligible for replay forever. The same identity-safe staging + # protocol works in the board directory when quarantine is unusable. + if board_descriptor is not None and source_stat is not None: + try: + retirement = _retire_terminal_entry( + path, + source_stat, + board_descriptor=board_descriptor, + quarantine_descriptor=( + quarantine_descriptor + if quarantine_descriptor is not None + else board_descriptor + ), + ) + except OSError: + retirement = "deferred" + print( + f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + finally: + if owns_source and source_descriptor is not None: + os.close(source_descriptor) + if quarantine_descriptor is not None: + os.close(quarantine_descriptor) + if owns_source and board_descriptor is not None: + os.close(board_descriptor) + print( + "quarantined terminal journal " + f"{path.name}: {reason}; identity={identity or 'unparseable'}; " + f"retirement={retirement}", + file=sys.stderr, + flush=True, + ) + return destination diff --git a/services/hermes/scripts/cli_lane_records.py b/services/hermes/scripts/cli_lane_records.py new file mode 100644 index 00000000..5229a069 --- /dev/null +++ b/services/hermes/scripts/cli_lane_records.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Bounded inode-pinned reads and validation for Hermes terminal records.""" + +from __future__ import annotations + +import json +import os +import stat +from pathlib import Path +from typing import Any + +import cli_lane_goal +from cli_lane_config import ( + MAX_TERMINAL_RECORD_BYTES, + QUARANTINE_HASH_BYTES, + RESULT_SCHEMA, + Route, + TerminalIdentity, + TerminalRecoverySnapshot, + TerminalSnapshot, + canonical_run_id, + utc_now, +) +from cli_lane_files import ( + _candidate_path, + _terminal_identity, + _terminal_path, + atomic_json, +) + + +def _read_bounded(descriptor: int, limit: int) -> bytes: + """Read at most ``limit`` bytes from a regular file descriptor.""" + chunks: list[bytes] = [] + remaining = max(0, limit) + while remaining: + chunk = os.read(descriptor, min(64 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + +def _open_small_json_snapshot(path: Path) -> TerminalSnapshot | None: + """Open and pin one bounded, singly-linked JSON document.""" + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + directory = None + descriptor = None + try: + directory = os.open(path.parent, directory_flags) + descriptor = os.open(path.name, file_flags, dir_fd=directory) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_size > MAX_TERMINAL_RECORD_BYTES + ): + return None + payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1) + if len(payload) > MAX_TERMINAL_RECORD_BYTES: + return None + current = os.stat( + path.name, + dir_fd=directory, + follow_symlinks=False, + ) + if ( + current.st_dev != opened.st_dev + or current.st_ino != opened.st_ino + or current.st_size != opened.st_size + or len(payload) != opened.st_size + ): + return None + value = json.loads(payload.decode("utf-8")) + if not isinstance(value, dict): + return None + snapshot = TerminalSnapshot(value, opened, descriptor, directory) + descriptor = None + directory = None + return snapshot + except (OSError, UnicodeError, json.JSONDecodeError): + return None + finally: + if descriptor is not None: + os.close(descriptor) + if directory is not None: + os.close(directory) + +def _open_terminal_recovery_snapshot(path: Path) -> TerminalRecoverySnapshot | None: + """Pin the entry recovery inspected, even when its payload is malformed.""" + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + directory = None + descriptor = None + try: + directory = os.open(path.parent, directory_flags) + observed = os.stat( + path.name, + dir_fd=directory, + follow_symlinks=False, + ) + if not stat.S_ISREG(observed.st_mode): + snapshot = TerminalRecoverySnapshot( + None, + observed, + None, + directory, + b"", + "non-regular", + ) + directory = None + return snapshot + try: + descriptor = os.open(path.name, file_flags, dir_fd=directory) + except OSError: + snapshot = TerminalRecoverySnapshot( + None, + observed, + None, + directory, + b"", + "open-failed", + ) + directory = None + return snapshot + opened = os.fstat(descriptor) + if opened.st_dev != observed.st_dev or opened.st_ino != observed.st_ino: + os.close(descriptor) + descriptor = None + snapshot = TerminalRecoverySnapshot( + None, + observed, + None, + directory, + b"", + "identity-changed-during-open", + ) + directory = None + return snapshot + observed = opened + if observed.st_nlink != 1: + snapshot = TerminalRecoverySnapshot( + None, + observed, + descriptor, + directory, + b"", + "hardlinked", + ) + descriptor = None + directory = None + return snapshot + if observed.st_size > MAX_TERMINAL_RECORD_BYTES: + prefix = _read_bounded(descriptor, QUARANTINE_HASH_BYTES) + snapshot = TerminalRecoverySnapshot( + None, + observed, + descriptor, + directory, + prefix, + "oversized", + ) + descriptor = None + directory = None + return snapshot + payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1) + after_read = os.fstat(descriptor) + unchanged = ( + after_read.st_dev == observed.st_dev + and after_read.st_ino == observed.st_ino + and after_read.st_size == observed.st_size + and after_read.st_mtime_ns == observed.st_mtime_ns + and after_read.st_ctime_ns == observed.st_ctime_ns + and len(payload) == observed.st_size + ) + document = None + invalid_reason = "unstable-payload" + if unchanged: + try: + parsed = json.loads(payload.decode("utf-8")) + if isinstance(parsed, dict): + document = parsed + invalid_reason = None + else: + invalid_reason = "non-object-payload" + except (UnicodeError, json.JSONDecodeError): + invalid_reason = "malformed-payload" + snapshot = TerminalRecoverySnapshot( + document, + after_read, + descriptor, + directory, + payload[:QUARANTINE_HASH_BYTES], + invalid_reason, + ) + descriptor = None + directory = None + return snapshot + except OSError: + return None + finally: + if descriptor is not None: + os.close(descriptor) + if directory is not None: + os.close(directory) + +def _open_terminal_snapshot( + path: Path, + identity: TerminalIdentity, +) -> TerminalSnapshot | None: + """Open a terminal journal only after its lexical identity is verified.""" + if _terminal_identity(path) != identity: + return None + return _open_small_json_snapshot(path) + +def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]: + """Read one small, singly-linked journal through its validated directory.""" + snapshot = _open_terminal_snapshot(path, identity) + if snapshot is None: + return {} + try: + return snapshot.document + finally: + snapshot.close() + +def _load_small_json(path: Path) -> dict[str, Any]: + """Read one bounded immutable evidence document without following links.""" + snapshot = _open_small_json_snapshot(path) + if snapshot is None: + return {} + try: + return snapshot.document + finally: + snapshot.close() + +def _persist_candidate( + state: dict[str, Any], + state_file: Path, + structured: dict[str, Any], + *, + route: Route, + returncode: int, + goal_turn: int, +) -> Path: + """Persist every structured response before any judge can supersede it.""" + sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1 + path = _candidate_path(state_file, state.get("run_id"), sequence) + while path.exists(): + sequence += 1 + path = _candidate_path(state_file, state.get("run_id"), sequence) + atomic_json( + path, + { + "board": state.get("board"), + "task_id": state.get("task_id"), + "expected_run_id": state.get("run_id"), + "goal_turn": goal_turn, + "provider": route.provider, + "model": route.model, + "effort": route.effort, + "returncode": returncode, + "structured": structured, + "recorded_at": utc_now(), + }, + ) + state["candidate_sequence"] = sequence + state["last_candidate_file"] = str(path) + atomic_json(state_file, state) + return path + +def _write_terminal_record( + state_file: Path, + *, + board: str, + task_id: str, + run_id: Any, + structured: dict[str, Any], + summary: str, + metadata: dict[str, Any], +) -> tuple[Path, dict[str, Any]]: + """Journal an accepted result before attempting the Kanban transaction.""" + if type(run_id) is not int or canonical_run_id(run_id) is None: + raise ValueError("terminal run ID must be a positive SQLite int64") + path = _terminal_path(state_file, run_id) + record = { + "board": board, + "task_id": task_id, + "expected_run_id": run_id, + "result": json.dumps(structured, sort_keys=True), + "summary": summary, + "metadata": metadata, + "kanban_state": "pending", + "recorded_at": utc_now(), + } + atomic_json(path, record) + return path, record + +def _terminal_record_valid( + record: dict[str, Any], + identity: TerminalIdentity | None = None, +) -> bool: + """Reject malformed or non-terminal replay journals without side effects.""" + if not isinstance(record, dict): + return False + required_strings = ("board", "task_id", "result", "summary") + if not all( + isinstance(record.get(key), str) and record[key] + for key in required_strings + ): + return False + expected_run_id = record.get("expected_run_id") + if ( + type(expected_run_id) is not int + or canonical_run_id(expected_run_id) is None + ): + return False + if not isinstance(record.get("metadata"), dict): + return False + try: + structured = json.loads(record["result"]) + except (TypeError, json.JSONDecodeError): + return False + if not isinstance(structured, dict): + return False + required = set(RESULT_SCHEMA["required"]) + if set(structured) != required: + return False + if ( + type(structured["status"]) is not str + or structured["status"] not in cli_lane_goal.RESULT_STATUSES + or type(structured["summary"]) is not str + or not structured["summary"].strip() + ): + return False + for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + value = structured[key] + if type(value) is not list or any(type(item) is not str for item in value): + return False + valid = ( + structured["status"] == "completed" + and structured["blockers"] == [] + and cli_lane_goal.unfinished_result_reason(structured) is None + and record["summary"] == structured["summary"] + and record.get("kanban_state") in {"pending", "prepared", "committed"} + ) + if not valid or identity is None: + return valid + return ( + record.get("board") == identity.board + and record.get("task_id") == identity.task_id + and record.get("expected_run_id") == identity.run_id + and record.get("kanban_state") == identity.state + ) diff --git a/services/hermes/scripts/cli_lane_recovery.py b/services/hermes/scripts/cli_lane_recovery.py new file mode 100644 index 00000000..8d692641 --- /dev/null +++ b/services/hermes/scripts/cli_lane_recovery.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +"""Bounded replay of pending, prepared, and staged terminal journals.""" + +from __future__ import annotations + +import hashlib +import os +import sys +from pathlib import Path +from typing import Any + +from cli_lane_board import _record_board_access_error +from cli_lane_config import ( + STATE_ROOT, + TerminalIdentity, + TerminalRecoverySnapshot, + canonical_run_id, +) +from cli_lane_evidence import ( + _persist_conflict_evidence, + _persist_prepared_evidence, + _same_terminal_document, + _terminal_evidence_path, + _terminal_evidence_valid, + _write_json_noreplace, +) +from cli_lane_files import ( + _LEGACY_RETIRE_NAME, + _RETIRE_NAME, + _SAFE_BOARD, + _terminal_evidence_identity, + _terminal_identity, + _terminal_path, + state_path, +) +from cli_lane_finalization import ( + _finalize_document_db, + _finalize_terminal_record, + _promote_prepared_evidence, + _recover_exact_run, + _resolve_pending_after_winner, + _retire_snapshot, + _retire_snapshot_after_db, +) +from cli_lane_quarantine import _quarantine_terminal +from cli_lane_records import ( + _load_small_json, + _load_terminal_json, + _open_terminal_recovery_snapshot, + _terminal_record_valid, +) + + +def _terminal_entry_absent(path: Path) -> bool: + """Confirm absence through a nofollow directory descriptor.""" + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + directory = None + try: + directory = os.open(path.parent, directory_flags) + os.stat(path.name, dir_fd=directory, follow_symlinks=False) + return False + except FileNotFoundError: + return True + except OSError: + return False + finally: + if directory is not None: + os.close(directory) + +def _staged_terminal_authority( + path: Path, + snapshot: TerminalRecoverySnapshot, +) -> tuple[TerminalIdentity, Path] | None: + """Bind a staged inode to the canonical pending name it was moved from.""" + try: + relative = path.relative_to(STATE_ROOT) + except ValueError: + return None + if len(relative.parts) == 2: + board, filename = relative.parts + elif len(relative.parts) == 3 and relative.parts[1] == "quarantine": + board, _quarantine, filename = relative.parts + else: + return None + match = _RETIRE_NAME.fullmatch(filename) + legacy = _LEGACY_RETIRE_NAME.fullmatch(filename) + document = snapshot.document + if ( + (match is None and legacy is None) + or document is None + or not _SAFE_BOARD.fullmatch(board) + or board in {".", ".."} + or document.get("board") != board + or not isinstance(document.get("task_id"), str) + or type(document.get("expected_run_id")) is not int + or canonical_run_id(document.get("expected_run_id")) is None + ): + return None + state = document.get("kanban_state") + if state not in {"pending", "prepared", "committed", "conflict"}: + return None + identity = TerminalIdentity( + board, document["task_id"], document["expected_run_id"], state + ) + if state in {"pending", "committed"}: + canonical = _terminal_path( + state_path(identity.board, identity.task_id), + identity.run_id, + state, + ) + valid = _terminal_record_valid(document, identity) + else: + canonical = _terminal_evidence_path(identity, state, document) + valid = _terminal_evidence_valid(document, identity, state) + board_dir = STATE_ROOT / board + if ( + canonical.parent != board_dir + or path.parent not in {board_dir, board_dir / "quarantine"} + or ( + match is not None + and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16] + != match.group("path_digest") + ) + or not valid + ): + return None + return identity, canonical + +def _retirement_staging_paths() -> list[Path]: + """List every directory where identity-safe retirement can stage an inode.""" + paths = list(STATE_ROOT.glob("*/.retire.*")) + paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*")) + return sorted(set(paths)) + +def _restore_staged_terminal_authority( + identity: TerminalIdentity, + canonical: Path, + document: dict[str, Any], +) -> None: + """Restore one hidden accepted artifact without executing it prematurely.""" + if identity.state == "pending": + _persist_prepared_evidence(identity, document) + return + if _write_json_noreplace(canonical, document): + return + existing = _load_small_json(canonical) + if identity.state in {"prepared", "conflict"}: + valid = _terminal_evidence_valid(existing, identity, identity.state) + else: + valid = _terminal_record_valid(existing, identity) + if valid and _same_terminal_document(existing, document): + return + if identity.state == "committed": + _persist_conflict_evidence( + identity, + document, + "staged committed evidence conflicts with the canonical first writer", + ) + return + raise OSError("staged terminal evidence path contains a conflicting result") + +def _recover_retirement_staging() -> int: + """Promote valid hidden journals to durable prepared evidence.""" + recovered = 0 + for path in _retirement_staging_paths(): + snapshot = _open_terminal_recovery_snapshot(path) + if snapshot is None: + continue + authority = _staged_terminal_authority(path, snapshot) + if authority is None: + try: + _quarantine_terminal( + path, + None, + "untrusted-retirement-staging", + snapshot=snapshot, + ) + finally: + snapshot.close() + continue + identity, canonical = authority + try: + _restore_staged_terminal_authority( + identity, + canonical, + snapshot.document or {}, + ) + retirement = _retire_snapshot( + path, + snapshot, + authority_name=canonical.name, + ) + except Exception as error: + _record_board_access_error(identity.board, error) + continue + finally: + snapshot.close() + if retirement in {"retired", "missing"}: + recovered += 1 + return recovered + +def _drain_retirement_staging() -> int: + """Converge bounded replacement churn without spinning on storage failure.""" + recovered = 0 + previous: tuple[tuple[str, int, int, int, int], ...] | None = None + for _attempt in range(16): + surface = [] + for path in _retirement_staging_paths(): + try: + observed = path.stat(follow_symlinks=False) + except OSError: + continue + surface.append( + ( + str(path), + observed.st_dev, + observed.st_ino, + observed.st_size, + observed.st_mtime_ns, + ) + ) + signature = tuple(surface) + if not signature or signature == previous: + break + previous = signature + recovered += _recover_retirement_staging() + return recovered + +def _recover_prepared_finalizations(kanban_db: Any) -> int: + """Recover an accepted result staged before an interrupted DB boundary.""" + recovered = 0 + recovered_identities: set[tuple[str, str, int]] = set() + prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json")) + + def durable_order(path: Path) -> tuple[int, str]: + try: + return path.stat(follow_symlinks=False).st_mtime_ns, str(path) + except OSError: + return 2**63 - 1, str(path) + + # When no DB winner exists yet, the first durably prepared result owns the + # run. The DB transaction remains the final arbiter across runner processes. + for path in sorted(prepared_paths, key=durable_order): + for _replacement_attempt in range(16): + identity = _terminal_evidence_identity(path) + snapshot = _open_terminal_recovery_snapshot(path) + if snapshot is None: + break + document = snapshot.document + if identity is None or identity.state != "prepared" or document is None: + try: + _quarantine_terminal( + path, + identity, + snapshot.invalid_reason or "malformed-prepared-evidence", + snapshot=snapshot, + ) + finally: + snapshot.close() + continue + expected = _terminal_evidence_path(identity, "prepared", document) + if ( + expected.name != path.name + or not _terminal_evidence_valid(document, identity, "prepared") + ): + try: + _quarantine_terminal( + path, + identity, + "foreign-prepared-evidence", + snapshot=snapshot, + ) + finally: + snapshot.close() + continue + try: + outcome = _finalize_document_db(kanban_db, identity, document) + except Exception as error: + _record_board_access_error(identity.board, error) + snapshot.close() + break + try: + if outcome == "committed": + _promote_prepared_evidence( + identity, + document, + path, + discard_prepared=False, + ) + _retire_snapshot_after_db(path, snapshot) + pending = _terminal_path( + state_path(identity.board, identity.task_id), + identity.run_id, + "pending", + ) + pending_identity = TerminalIdentity( + identity.board, + identity.task_id, + identity.run_id, + "pending", + ) + _resolve_pending_after_winner( + pending, + pending_identity, + document, + ) + recovered_identity = ( + identity.board, + identity.task_id, + identity.run_id, + ) + if recovered_identity not in recovered_identities: + recovered_identities.add(recovered_identity) + recovered += 1 + elif outcome == "stale": + _persist_conflict_evidence( + identity, + document, + "prepared result no longer matches the authoritative task run", + ) + _retire_snapshot_after_db(path, snapshot) + else: + snapshot.close() + break + except Exception as error: + _record_board_access_error(identity.board, error) + snapshot.close() + break + snapshot.close() + return recovered + +def recover_pending_finalizations() -> int: + """Replay accepted exact-run results before scheduling more provider work.""" + from hermes_cli import kanban_db + + _drain_retirement_staging() + recovered = _recover_prepared_finalizations(kanban_db) + for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")): + identity = _terminal_identity(path) + last_invalid_reason = "malformed-name" if identity is None else None + retired_invalid = False + for _replacement_attempt in range(16): + snapshot = _open_terminal_recovery_snapshot(path) + if snapshot is None: + if ( + identity is not None + and retired_invalid + and _terminal_entry_absent(path) + ): + _recover_exact_run( + kanban_db, + identity, + last_invalid_reason or "malformed-payload", + ) + break + record = snapshot.document or {} + reason = None + if identity is None: + reason = "malformed-name" + elif not _terminal_record_valid(record): + reason = snapshot.invalid_reason or "malformed-payload" + elif not _terminal_record_valid(record, identity): + reason = "foreign-identity" + if reason is not None: + try: + _quarantine_terminal( + path, + identity, + reason, + snapshot=snapshot, + ) + finally: + snapshot.close() + retired_invalid = True + last_invalid_reason = reason + continue + assert identity is not None + try: + outcome = _finalize_terminal_record( + kanban_db, + path, + record, + snapshot=snapshot, + ) + except Exception as error: + _record_board_access_error(identity.board, error) + break + if outcome == "committed": + recovered += 1 + break + if outcome in {"invalid", "foreign"}: + # The pathname changed after the recovery snapshot closed. + # Reclassify and pin the new inode instead of quarantining it + # using authority derived from the older entry. + last_invalid_reason = outcome + continue + break + else: + print( + f"terminal journal replacement churn deferred for {path.name}", + file=sys.stderr, + flush=True, + ) + # A quarantine or identity-safe retirement performed above can itself + # encounter replacement churn. Promote every resulting hidden generation + # back to first-class prepared/conflict/committed evidence before this + # recovery pass returns; DB replay, if still needed, occurs on the next + # bounded dispatcher pass. + _drain_retirement_staging() + return recovered + +def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool: + """Keep an exact run claimed while its accepted result awaits replay.""" + if type(run_id) is not int or canonical_run_id(run_id) is None: + return False + path = _terminal_path(state_path(board, task_id), run_id, "pending") + try: + path.stat() + except FileNotFoundError: + pass + except OSError: + return False + else: + identity = _terminal_identity(path) + if identity is not None: + record = _load_terminal_json(path, identity) + if _terminal_record_valid(record, identity): + return True + base = state_path(board, task_id) + for prepared in base.parent.glob( + f"{base.stem}.run-{run_id}.terminal.prepared-*.json" + ): + identity = _terminal_evidence_identity(prepared) + if identity is None or identity.state != "prepared": + continue + record = _load_small_json(prepared) + if _terminal_evidence_valid(record, identity, "prepared"): + return True + staged_paths = list(base.parent.glob(".retire.*")) + quarantine = base.parent / "quarantine" + if quarantine.is_dir() and not quarantine.is_symlink(): + staged_paths.extend(quarantine.glob(".retire.*")) + for staged in staged_paths: + snapshot = _open_terminal_recovery_snapshot(staged) + if snapshot is None: + continue + try: + authority = _staged_terminal_authority(staged, snapshot) + if ( + authority is not None + and authority[0].board == board + and authority[0].task_id == task_id + and authority[0].run_id == run_id + and authority[0].state in {"pending", "prepared", "committed"} + ): + return True + finally: + snapshot.close() + return False diff --git a/services/hermes/scripts/cli_lane_retention.py b/services/hermes/scripts/cli_lane_retention.py new file mode 100644 index 00000000..c88df653 --- /dev/null +++ b/services/hermes/scripts/cli_lane_retention.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Bounded age, count, and byte retention for non-pending lane artifacts.""" + +from __future__ import annotations + +import os +import stat +import sys +import time +from pathlib import Path + +from cli_lane_config import ( + ARTIFACT_GC_INTERVAL_SECONDS, + ARTIFACT_RETENTION_AGE_SECONDS, + ARTIFACT_RETENTION_BYTES, + ARTIFACT_RETENTION_COUNT, + STATE_ROOT, +) +from cli_lane_evidence import _retire_terminal_entry, _terminal_evidence_valid +from cli_lane_files import ( + _fsync_directory, + _terminal_evidence_identity, + _terminal_identity, +) +from cli_lane_quarantine import _quarantine_terminal +from cli_lane_recovery import _staged_terminal_authority +from cli_lane_records import _open_terminal_recovery_snapshot, _terminal_record_valid + + +def _artifact_gc_candidates(board_dir: Path) -> list[Path]: + """List only bounded artifact classes; pending journals are excluded.""" + candidates: list[Path] = [] + patterns = ( + "*.provider-*.result.json", + "*.candidate-*.json", + "*.terminal.committed.json", + "*.terminal.conflict-*.json", + ".retire.*", + ) + for pattern in patterns: + candidates.extend(board_dir.glob(pattern)) + quarantine = board_dir / "quarantine" + if quarantine.is_dir(): + candidates.extend(quarantine.glob("*.quarantine")) + candidates.extend(quarantine.glob(".retire.*")) + return candidates + +def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool: + """Remove only the exact artifact inode selected by retention.""" + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = None + try: + descriptor = os.open(path.parent, directory_flags) + outcome = _retire_terminal_entry( + path, + observed, + board_descriptor=descriptor, + quarantine_descriptor=descriptor, + ) + return outcome == "retired" + except OSError: + return False + finally: + if descriptor is not None: + os.close(descriptor) + + +def _quarantine_invalid_retained_terminal(path: Path) -> tuple[bool, bool]: + """Audit retained committed/conflict evidence on the bounded GC cadence.""" + committed = path.name.endswith(".terminal.committed.json") + conflict = ".terminal.conflict-" in path.name and path.name.endswith(".json") + if not committed and not conflict: + return False, False + identity = ( + _terminal_identity(path) if committed else _terminal_evidence_identity(path) + ) + snapshot = _open_terminal_recovery_snapshot(path) + if snapshot is None: + return True, False + document = snapshot.document + valid = False + if identity is not None and document is not None: + valid = ( + _terminal_record_valid(document, identity) + if committed + else _terminal_evidence_valid(document, identity, "conflict") + ) + if valid: + snapshot.close() + return False, False + try: + _quarantine_terminal( + path, + identity, + snapshot.invalid_reason or "invalid-retained-terminal-evidence", + snapshot=snapshot, + ) + finally: + snapshot.close() + try: + path.stat(follow_symlinks=False) + except FileNotFoundError: + return True, True + except OSError: + return True, False + return True, False + +def gc_lane_artifacts( + *, + now: float | None = None, + max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS, + max_count: int = ARTIFACT_RETENTION_COUNT, + max_bytes: int = ARTIFACT_RETENTION_BYTES, +) -> int: + """Bound non-pending lane artifacts by age, count, and bytes per board.""" + current = time.time() if now is None else now + removed = 0 + for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else (): + if not board_dir.is_dir() or board_dir.is_symlink(): + continue + board_removed = 0 + entries: list[tuple[Path, os.stat_result]] = [] + for path in _artifact_gc_candidates(board_dir): + classified, quarantined = _quarantine_invalid_retained_terminal(path) + if classified: + if quarantined: + removed += 1 + board_removed += 1 + continue + if path.name.startswith(".retire."): + snapshot = _open_terminal_recovery_snapshot(path) + if snapshot is not None: + try: + if _staged_terminal_authority(path, snapshot) is not None: + # Accepted authority is first promoted/replayed by + # recovery; retention never deletes it directly. + continue + finally: + snapshot.close() + try: + file_stat = path.stat(follow_symlinks=False) + except OSError: + continue + if not stat.S_ISREG(file_stat.st_mode): + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 + continue + if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds: + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 + else: + entries.append((path, file_stat)) + retained_count = 0 + retained_bytes = 0 + for path, file_stat in sorted( + entries, key=lambda item: item[1].st_mtime, reverse=True + ): + exceeds_count = max_count >= 0 and retained_count >= max_count + exceeds_bytes = ( + max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes + ) + if exceeds_count or exceeds_bytes: + if _unlink_artifact_if_same(path, file_stat): + removed += 1 + board_removed += 1 + continue + retained_count += 1 + retained_bytes += file_stat.st_size + if board_removed: + _fsync_directory(board_dir) + return removed + +LAST_ARTIFACT_GC = 0.0 + +def maybe_gc_lane_artifacts(*, now: float | None = None) -> int: + """Run artifact retention on a bounded cadence, not every dispatcher tick.""" + global LAST_ARTIFACT_GC + current = time.time() if now is None else now + if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS: + return 0 + LAST_ARTIFACT_GC = current + try: + return gc_lane_artifacts(now=current) + except OSError as error: + print( + f"lane artifact retention deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + return 0 diff --git a/services/hermes/scripts/cli_lane_routing.py b/services/hermes/scripts/cli_lane_routing.py new file mode 100644 index 00000000..3ee58d34 --- /dev/null +++ b/services/hermes/scripts/cli_lane_routing.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Switchyard route selection and provider-health exclusion for CLI workers.""" + +from __future__ import annotations + +import json +import re +import time +import urllib.error +import urllib.request +from typing import Any, Callable + +from cli_lane_config import ( + EFFORTS, + PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS, + PROVIDER_HEALTH_MAX_AGE_SECONDS, + PROVIDER_HEALTH_PATHS, + SWITCHYARD_URL, + Route, +) +from cli_lane_files import load_json + + +def parse_assignee(assignee: str) -> tuple[str | None, str | None]: + """Parse external lane overrides while leaving cli-auto fully automatic.""" + value = str(assignee or "").strip().lower() + if value == "cli-auto": + return None, None + match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value) + if not match: + raise ValueError(f"unsupported external lane assignee: {assignee}") + return match.group(1), match.group(2) + +def fresh_unavailable_provider(now: float | None = None) -> str | None: + """Return one recently proven-down provider for automatic route exclusion.""" + current = time.time() if now is None else now + unavailable: list[str] = [] + for provider, path in PROVIDER_HEALTH_PATHS.items(): + health = load_json(path) + try: + age = current - path.stat().st_mtime + except OSError: + continue + max_age = ( + PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS + if health.get("authenticated") is False + else PROVIDER_HEALTH_MAX_AGE_SECONDS + ) + if ( + 0 <= age <= max_age + and health.get("state") == "unavailable" + ): + unavailable.append(provider) + # If both providers are down, keep the normal attempt/fallback path so the + # card records authoritative current errors instead of trusting snapshots. + return unavailable[0] if len(unavailable) == 1 else None + +def _decode_worker_target(value: str) -> tuple[str, str, str]: + """Decode the selected model header emitted by a worker decision route.""" + parts = value.split("/", 3) + if len(parts) != 4 or parts[0] != "worker": + raise RuntimeError(f"invalid Switchyard worker target: {value}") + provider, model, effort = parts[1:] + if provider not in {"codex", "claude"} or effort not in EFFORTS: + raise RuntimeError(f"unsupported Switchyard worker target: {value}") + return provider, model, effort + +def select_route( + prompt: str, + assignee: str, + *, + exclude_provider: str | None = None, + exclude_reason: str | None = None, + switchyard_url: str = SWITCHYARD_URL, + open_request: Callable[..., Any] = urllib.request.urlopen, +) -> Route: + """Ask Switchyard to select one native CLI worker at this boundary.""" + started = time.monotonic() + manual_provider, manual_effort = parse_assignee(assignee) + if manual_provider and manual_effort: + route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}" + source = "switchyard-manual" + else: + route_id = "atlas/worker/auto/maximum" + source = "switchyard-classifier" + context = prompt + if exclude_provider: + reason = exclude_reason or "failed or exhausted capacity at this boundary" + context += ( + f"\n\nRouting constraint: the {exclude_provider} provider {reason}. " + "Do not select it." + ) + payload = json.dumps( + { + "model": route_id, + "messages": [{"role": "user", "content": context}], + "stream": False, + "max_tokens": 1, + } + ).encode("utf-8") + request = urllib.request.Request( + switchyard_url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with open_request(request, timeout=60) as response: + selected = str(response.headers.get("x-model-router-selected-model") or "") + rationale = str(response.headers.get("x-model-router-rationale") or "") + response_body = response.read() + except (OSError, urllib.error.URLError) as exc: + raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc + provider, model, effort = _decode_worker_target(selected) + # Switchyard preserves the stable tier target in the selection header and + # top-level response model. The worker broker's assistant content contains + # the steward-resolved provider model required by the native CLI. + try: + response_document = json.loads(response_body) + resolved_target = str(response_document["choices"][0]["message"]["content"] or "") + resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target) + required_prefix = "gpt-" if provider == "codex" else "claude-" + if ( + resolved_provider == provider + and resolved_effort == effort + and resolved_model.startswith(required_prefix) + ): + model = resolved_model + except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + pass + if exclude_provider and provider == exclude_provider: + alternate = "claude" if provider == "codex" else "codex" + guarded = select_route( + prompt, + f"cli-{alternate}-{effort}", + switchyard_url=switchyard_url, + open_request=open_request, + ) + return Route( + provider=guarded.provider, + model=guarded.model, + effort=guarded.effort, + profile=guarded.profile, + classifier=f"{source}-health-guard", + reason=( + f"Switchyard selected excluded {exclude_provider}; preserved " + f"its {effort} effort on healthy-provider route. {guarded.reason}" + ), + latency_ms=int((time.monotonic() - started) * 1000), + fallback_chain=(), + ) + return Route( + provider=provider, + model=model, + effort=effort, + profile=f"{provider}-{effort}", + classifier=source, + reason=rationale or f"Switchyard selected {selected}", + latency_ms=int((time.monotonic() - started) * 1000), + fallback_chain=(), + ) diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 30c84daf..5a51e928 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -1,2243 +1,131 @@ #!/usr/bin/env python3 +# ruff: noqa: F401 """Run durable Codex and Claude workers from Hermes' authoritative Kanban.""" from __future__ import annotations -import concurrent.futures -import ctypes -import errno -import hashlib -import json +# The facade keeps the deployed entry point and compatibility API stable while +# domain modules own each independently testable reliability boundary. import os -import re -import selectors -import signal import sqlite3 -import stat import subprocess -import sys -import threading -import time import uuid -import urllib.error -import urllib.request -from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Callable import cli_lane_goal - - -DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) -SWITCHYARD_URL = os.environ.get( - "HERMES_SWITCHYARD_URL", - "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions", +from cli_lane_board import ( + _board_call, + _external, + _record_board_access_error, + _resolve_workspace, + _task_context, + _task_value, ) -STATE_ROOT = DATA_ROOT / "cli-lanes" -CODEX_BIN = DATA_ROOT / "tools/bin/codex" -CLAUDE_BIN = DATA_ROOT / "tools/bin/claude" -CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json" -RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json" -EFFORTS = ("low", "medium", "high", "xhigh") -EXTERNAL_PREFIX = "cli-" -DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60 -DEFAULT_MAX_RUNTIME = 12 * 60 * 60 -HEARTBEAT_SECONDS = 20 -PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 -PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60 -KANBAN_STORAGE_ATTEMPTS = 5 -ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60 -ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60 -ARTIFACT_RETENTION_COUNT = 256 -ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024 -MAX_TERMINAL_RECORD_BYTES = 1024 * 1024 -QUARANTINE_HASH_BYTES = 64 * 1024 -PROVIDER_HEALTH_PATHS = { - "codex": DATA_ROOT / "provider-health/codex.json", - "claude": DATA_ROOT / "provider-health/claude.json", -} -WORKTREE_LOCK = threading.Lock() -BOARD_CORRUPTION_ERRORS: dict[str, str] = {} -LAST_ARTIFACT_GC = 0.0 -CAPACITY_PATTERN = re.compile( - r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", - re.I, +from cli_lane_config import ( + ARTIFACT_GC_INTERVAL_SECONDS, + ARTIFACT_RETENTION_AGE_SECONDS, + ARTIFACT_RETENTION_BYTES, + ARTIFACT_RETENTION_COUNT, + BOARD_CORRUPTION_ERRORS, + CLAUDE_BIN, + CODEX_BIN, + DATA_ROOT, + DEFAULT_CLAIM_TTL, + DEFAULT_MAX_RUNTIME, + MAX_TERMINAL_RECORD_BYTES, + PROVIDER_HEALTH_PATHS, + QUARANTINE_HASH_BYTES, + RESULT_SCHEMA, + RESULT_SCHEMA_PATH, + STATE_ROOT, + ProcessResult, + Route, + TerminalFinalizationPending, + TerminalIdentity, + canonical_run_id, + utc_now, ) -NO_CLAUDE_SESSION = "No conversation found with session ID:" -CLAUDE_SESSION_COLLISION = "Session ID already in use" -NO_CODEX_THREAD = "no rollout found for thread id" - - -RESULT_SCHEMA: dict[str, Any] = { - "type": "object", - "additionalProperties": False, - "required": [ - "status", - "summary", - "changed_files", - "tests_run", - "artifacts", - "findings", - "blockers", - ], - "properties": { - "status": { - "type": "string", - "enum": sorted(cli_lane_goal.RESULT_STATUSES), - }, - "summary": {"type": "string"}, - "changed_files": {"type": "array", "items": {"type": "string"}}, - "tests_run": {"type": "array", "items": {"type": "string"}}, - "artifacts": {"type": "array", "items": {"type": "string"}}, - "findings": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Defects, risks, or observations discovered by a review or diagnosis. " - "Findings do not prevent the assigned review or diagnosis from completing." - ), - }, - "blockers": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Concrete obstacles that prevent completion of the assigned task itself. " - "This must be empty when status is completed; review findings belong in findings." - ), - }, - }, -} - - -@dataclass(frozen=True) -class Route: - """One provider/model/effort decision for an observable worker boundary.""" - - provider: str - model: str - effort: str - profile: str - classifier: str - reason: str - latency_ms: int - fallback_chain: tuple[str, ...] - - -@dataclass -class ProcessResult: - """Captured outcome from one provider CLI invocation.""" - - returncode: int - output: str - structured: dict[str, Any] | None - capacity_failure: bool - - -class TerminalFinalizationPending(RuntimeError): - """An accepted worker result is durable but not committed to Kanban yet.""" - - -@dataclass(frozen=True) -class TerminalIdentity: - """Replay authority derived only from a journal's directory and filename.""" - - board: str - task_id: str - run_id: int - state: str - - -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: - """Durably replace a small state document without following temp symlinks.""" - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - os.chmod(path.parent, 0o700, follow_symlinks=False) - temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - flags |= getattr(os, "O_NOFOLLOW", 0) - descriptor = None - try: - descriptor = os.open(temporary, flags, mode) - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - descriptor = None - json.dump(value, stream, indent=2, sort_keys=True) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - os.chmod(temporary, mode, follow_symlinks=False) - os.replace(temporary, path) - _fsync_directory(path.parent) - finally: - if descriptor is not None: - os.close(descriptor) - try: - temporary.unlink() - except FileNotFoundError: - pass - - -def _fsync_directory(directory: Path) -> None: - """Persist directory-entry changes after an atomic rename or quarantine.""" - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - flags |= getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(directory, flags) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -def load_json(path: Path) -> dict[str, Any]: - try: - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(path, flags) - with os.fdopen(descriptor, "r", encoding="utf-8") as stream: - value = json.load(stream) - except (OSError, UnicodeError, json.JSONDecodeError): - return {} - return value if isinstance(value, dict) else {} - - -def parse_assignee(assignee: str) -> tuple[str | None, str | None]: - """Parse external lane overrides while leaving cli-auto fully automatic.""" - value = str(assignee or "").strip().lower() - if value == "cli-auto": - return None, None - match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value) - if not match: - raise ValueError(f"unsupported external lane assignee: {assignee}") - return match.group(1), match.group(2) - - -def fresh_unavailable_provider(now: float | None = None) -> str | None: - """Return one recently proven-down provider for automatic route exclusion.""" - current = time.time() if now is None else now - unavailable: list[str] = [] - for provider, path in PROVIDER_HEALTH_PATHS.items(): - health = load_json(path) - try: - age = current - path.stat().st_mtime - except OSError: - continue - max_age = ( - PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS - if health.get("authenticated") is False - else PROVIDER_HEALTH_MAX_AGE_SECONDS - ) - if ( - 0 <= age <= max_age - and health.get("state") == "unavailable" - ): - unavailable.append(provider) - # If both providers are down, keep the normal attempt/fallback path so the - # card records authoritative current errors instead of trusting snapshots. - return unavailable[0] if len(unavailable) == 1 else None - - -def _decode_worker_target(value: str) -> tuple[str, str, str]: - """Decode the selected model header emitted by a worker decision route.""" - parts = value.split("/", 3) - if len(parts) != 4 or parts[0] != "worker": - raise RuntimeError(f"invalid Switchyard worker target: {value}") - provider, model, effort = parts[1:] - if provider not in {"codex", "claude"} or effort not in EFFORTS: - raise RuntimeError(f"unsupported Switchyard worker target: {value}") - return provider, model, effort - - -def select_route( - prompt: str, - assignee: str, - *, - exclude_provider: str | None = None, - exclude_reason: str | None = None, - switchyard_url: str = SWITCHYARD_URL, - open_request: Callable[..., Any] = urllib.request.urlopen, -) -> Route: - """Ask Switchyard to select one native CLI worker at this boundary.""" - started = time.monotonic() - manual_provider, manual_effort = parse_assignee(assignee) - if manual_provider and manual_effort: - route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}" - source = "switchyard-manual" - else: - route_id = "atlas/worker/auto/maximum" - source = "switchyard-classifier" - context = prompt - if exclude_provider: - reason = exclude_reason or "failed or exhausted capacity at this boundary" - context += ( - f"\n\nRouting constraint: the {exclude_provider} provider {reason}. " - "Do not select it." - ) - payload = json.dumps( - { - "model": route_id, - "messages": [{"role": "user", "content": context}], - "stream": False, - "max_tokens": 1, - } - ).encode("utf-8") - request = urllib.request.Request( - switchyard_url, - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with open_request(request, timeout=60) as response: - selected = str(response.headers.get("x-model-router-selected-model") or "") - rationale = str(response.headers.get("x-model-router-rationale") or "") - response_body = response.read() - except (OSError, urllib.error.URLError) as exc: - raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc - provider, model, effort = _decode_worker_target(selected) - # Switchyard preserves the stable tier target in the selection header and - # top-level response model. The worker broker's assistant content contains - # the steward-resolved provider model required by the native CLI. - try: - response_document = json.loads(response_body) - resolved_target = str(response_document["choices"][0]["message"]["content"] or "") - resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target) - required_prefix = "gpt-" if provider == "codex" else "claude-" - if ( - resolved_provider == provider - and resolved_effort == effort - and resolved_model.startswith(required_prefix) - ): - model = resolved_model - except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): - pass - if exclude_provider and provider == exclude_provider: - alternate = "claude" if provider == "codex" else "codex" - guarded = select_route( - prompt, - f"cli-{alternate}-{effort}", - switchyard_url=switchyard_url, - open_request=open_request, - ) - return Route( - provider=guarded.provider, - model=guarded.model, - effort=guarded.effort, - profile=guarded.profile, - classifier=f"{source}-health-guard", - reason=( - f"Switchyard selected excluded {exclude_provider}; preserved " - f"its {effort} effort on healthy-provider route. {guarded.reason}" - ), - latency_ms=int((time.monotonic() - started) * 1000), - fallback_chain=(), - ) - return Route( - provider=provider, - model=model, - effort=effort, - profile=f"{provider}-{effort}", - classifier=source, - reason=rationale or f"Switchyard selected {selected}", - latency_ms=int((time.monotonic() - started) * 1000), - fallback_chain=(), - ) - - -def state_path(board: str, task_id: str) -> Path: - safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board) - safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id) - return STATE_ROOT / safe_board / f"{safe_task}.json" - - -def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path: - """Return a unique provider-result path; completed turns are never reused.""" - safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) - return state_file.with_name( - f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json" - ) - - -def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path: - """Return the durable path for one exact structured worker response.""" - safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) - return state_file.with_name( - f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json" - ) - - -def _terminal_path( - state_file: Path, - run_id: Any, - state: str = "pending", -) -> Path: - """Return the identity-bound replay path for an accepted terminal response.""" - if state not in {"pending", "committed"}: - raise ValueError(f"invalid terminal journal state: {state}") - safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown")) - return state_file.with_name( - f"{state_file.stem}.run-{safe_run}.terminal.{state}.json" - ) - - -_TERMINAL_NAME = re.compile( - r"^(?P[a-zA-Z0-9_.-]+)\.run-(?P[0-9]+)\." - r"terminal\.(?Ppending|committed)\.json$" +from cli_lane_dispatch import claim_ready, main, recover_orphans +from cli_lane_evidence import ( + _persist_conflict_evidence, + _persist_prepared_evidence, + _rename_noreplace, + _retire_terminal_entry, + _same_terminal_document, + _terminal_document_digest, + _terminal_evidence_path, + _terminal_evidence_valid, + _write_json_noreplace, ) -_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$") - - -def _terminal_identity(path: Path) -> TerminalIdentity | None: - """Parse replay authority lexically before opening a journal or board.""" - try: - relative = path.relative_to(STATE_ROOT) - except ValueError: - return None - if len(relative.parts) != 2: - return None - board, filename = relative.parts - match = _TERMINAL_NAME.fullmatch(filename) - try: - board_stat = path.parent.stat(follow_symlinks=False) - except OSError: - return None - if ( - not match - or not stat.S_ISDIR(board_stat.st_mode) - or not _SAFE_BOARD.fullmatch(board) - or board in {".", ".."} - or match.group("task") in {".", ".."} - ): - return None - return TerminalIdentity( - board=board, - task_id=match.group("task"), - run_id=int(match.group("run")), - state=match.group("state"), - ) - - -def _read_bounded(descriptor: int, limit: int) -> bytes: - """Read at most ``limit`` bytes from a regular file descriptor.""" - chunks: list[bytes] = [] - remaining = max(0, limit) - while remaining: - chunk = os.read(descriptor, min(64 * 1024, remaining)) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - return b"".join(chunks) - - -def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]: - """Read one small, singly-linked journal through its validated directory.""" - if _terminal_identity(path) != identity: - return {} - directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - directory_flags |= getattr(os, "O_NOFOLLOW", 0) - file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - directory = None - descriptor = None - try: - directory = os.open(path.parent, directory_flags) - descriptor = os.open(path.name, file_flags, dir_fd=directory) - opened = os.fstat(descriptor) - if ( - not stat.S_ISREG(opened.st_mode) - or opened.st_nlink != 1 - or opened.st_size > MAX_TERMINAL_RECORD_BYTES - ): - return {} - payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1) - if len(payload) > MAX_TERMINAL_RECORD_BYTES: - return {} - current = os.stat( - path.name, - dir_fd=directory, - follow_symlinks=False, - ) - if ( - current.st_dev != opened.st_dev - or current.st_ino != opened.st_ino - or current.st_size != opened.st_size - or len(payload) != opened.st_size - ): - return {} - value = json.loads(payload.decode("utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError): - return {} - finally: - if descriptor is not None: - os.close(descriptor) - if directory is not None: - os.close(directory) - return value if isinstance(value, dict) else {} - - -def _persist_candidate( - state: dict[str, Any], - state_file: Path, - structured: dict[str, Any], - *, - route: Route, - returncode: int, - goal_turn: int, -) -> Path: - """Persist every structured response before any judge can supersede it.""" - sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1 - path = _candidate_path(state_file, state.get("run_id"), sequence) - while path.exists(): - sequence += 1 - path = _candidate_path(state_file, state.get("run_id"), sequence) - atomic_json( - path, - { - "board": state.get("board"), - "task_id": state.get("task_id"), - "expected_run_id": state.get("run_id"), - "goal_turn": goal_turn, - "provider": route.provider, - "model": route.model, - "effort": route.effort, - "returncode": returncode, - "structured": structured, - "recorded_at": utc_now(), - }, - ) - state["candidate_sequence"] = sequence - state["last_candidate_file"] = str(path) - atomic_json(state_file, state) - return path - - -def _write_terminal_record( - state_file: Path, - *, - board: str, - task_id: str, - run_id: Any, - structured: dict[str, Any], - summary: str, - metadata: dict[str, Any], -) -> tuple[Path, dict[str, Any]]: - """Journal an accepted result before attempting the Kanban transaction.""" - path = _terminal_path(state_file, run_id) - record = { - "board": board, - "task_id": task_id, - "expected_run_id": run_id, - "result": json.dumps(structured, sort_keys=True), - "summary": summary, - "metadata": metadata, - "kanban_state": "pending", - "recorded_at": utc_now(), - } - atomic_json(path, record) - return path, record - - -def _terminal_record_valid( - record: dict[str, Any], - identity: TerminalIdentity | None = None, -) -> bool: - """Reject malformed or non-terminal replay journals without side effects.""" - if not isinstance(record, dict): - return False - required_strings = ("board", "task_id", "result", "summary") - if not all( - isinstance(record.get(key), str) and record[key] - for key in required_strings - ): - return False - if not isinstance(record.get("expected_run_id"), int): - return False - if not isinstance(record.get("metadata"), dict): - return False - try: - structured = json.loads(record["result"]) - except (TypeError, json.JSONDecodeError): - return False - if not isinstance(structured, dict): - return False - required = set(RESULT_SCHEMA["required"]) - if set(structured) != required: - return False - if ( - type(structured["status"]) is not str - or structured["status"] not in cli_lane_goal.RESULT_STATUSES - or type(structured["summary"]) is not str - or not structured["summary"].strip() - ): - return False - for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): - value = structured[key] - if type(value) is not list or any(type(item) is not str for item in value): - return False - valid = ( - structured["status"] == "completed" - and structured["blockers"] == [] - and cli_lane_goal.unfinished_result_reason(structured) is None - and record["summary"] == structured["summary"] - and record.get("kanban_state") in {"pending", "committed"} - ) - if not valid or identity is None: - return valid - return ( - record.get("board") == identity.board - and record.get("task_id") == identity.task_id - and record.get("expected_run_id") == identity.run_id - ) - - -def build_prompt(context: str, workspace: Path, handoff: str = "") -> str: - """Create a bounded worker contract with an explicit machine-readable result.""" - return f"""You are a durable coding worker managed by Hermes Kanban. - -Work only on this objective and its acceptance criteria: - -{context} - -Workspace: {workspace} -{handoff} - -Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials. - -Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly. -""" - - -def git_handoff(workspace: Path, prior_output: str) -> str: - """Build an explicit cross-provider handoff without transferring hidden state.""" - def read_git(*args: str) -> str: - completed = subprocess.run( - ["git", "-C", str(workspace), *args], - text=True, - capture_output=True, - timeout=30, - check=False, - ) - return (completed.stdout or completed.stderr).strip()[-12000:] - - return ( - "\nCross-provider handoff from a failed or exhausted worker:\n" - f"Git status:\n{read_git('status', '--short', '--branch')}\n\n" - f"Current diff summary:\n{read_git('diff', '--stat')}\n\n" - f"Prior worker tail:\n{prior_output[-10000:]}\n" - "Reinspect the workspace and verify all inherited claims before continuing." - ) - - -def workspace_artifacts(workspace: Path, values: Any) -> list[str]: - """Return existing regular artifacts contained by the task worktree.""" - if not isinstance(values, list): - return [] - root = workspace.resolve() - artifacts: list[str] = [] - for value in values: - if not isinstance(value, str) or not value.strip(): - continue - candidate = Path(value.strip()).expanduser() - if not candidate.is_absolute(): - candidate = root / candidate - try: - resolved = candidate.resolve(strict=True) - resolved.relative_to(root) - except (OSError, RuntimeError, ValueError): - continue - if resolved.is_file(): - artifacts.append(str(resolved)) - return list(dict.fromkeys(artifacts)) - - -def _extract_json(value: Any) -> dict[str, Any] | None: - if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES: - return value - if not isinstance(value, str): - return None - candidates = [value] - match = re.search(r"\{.*\}", value, re.DOTALL) - if match: - candidates.append(match.group(0)) - for candidate in candidates: - try: - parsed = json.loads(candidate) - except json.JSONDecodeError: - continue - if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES: - return parsed - return None - - -def _event_payload(provider: str, line: str, state: dict[str, Any], state_file: Path) -> dict[str, Any] | None: - """Persist provider session identifiers before interpreting final output.""" - try: - event = json.loads(line) - except json.JSONDecodeError: - return None - if not isinstance(event, dict): - return None - if provider == "codex" and event.get("type") == "thread.started": - thread_id = event.get("thread_id") or event.get("thread", {}).get("id") - if thread_id: - state["codex_thread_id"] = str(thread_id) - atomic_json(state_file, state) - if provider == "claude" and event.get("session_id"): - state["claude_session_id"] = str(event["session_id"]) - atomic_json(state_file, state) - for key in ("structured_output", "result", "output", "text"): - parsed = _extract_json(event.get(key)) - if parsed: - return parsed - item = event.get("item") - if isinstance(item, dict): - for key in ("text", "content"): - parsed = _extract_json(item.get(key)) - if parsed: - return parsed - return None - - -def stream_process( - command: list[str], - *, - provider: str, - cwd: Path, - env: dict[str, str], - log_path: Path, - state: dict[str, Any], - state_file: Path, - heartbeat: Callable[[str], bool], - max_runtime: int, -) -> ProcessResult: - """Stream JSONL to Kanban logs while maintaining the authoritative lease.""" - process = subprocess.Popen( - command, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - start_new_session=True, - ) - assert process.stdout is not None - selector = selectors.DefaultSelector() - selector.register(process.stdout, selectors.EVENT_READ) - started = time.monotonic() - last_heartbeat = 0.0 - known_descendants: dict[int, tuple[int, int]] = {} - lines: list[str] = [] - structured: dict[str, Any] | None = None - log_path.parent.mkdir(parents=True, exist_ok=True) - forced_failure = "" - with log_path.open("a", encoding="utf-8") as log: - log.write(f"\n[{utc_now()}] starting {provider} worker\n") - log.flush() - while process.poll() is None: - known_descendants.update(_descendant_processes(process.pid)) - now = time.monotonic() - if now - started > max_runtime: - _terminate_worker_process(process, known_descendants) - forced_failure = "worker exceeded its maximum runtime" - lines.append(forced_failure + "\n") - break - if now - last_heartbeat >= HEARTBEAT_SECONDS: - if not heartbeat(f"{provider} worker active for {round(now - started)}s"): - _terminate_worker_process(process, known_descendants) - forced_failure = "Kanban lease was lost; provider process terminated" - lines.append(forced_failure + "\n") - break - last_heartbeat = now - for key, _ in selector.select(timeout=1.0): - line = key.fileobj.readline() - if not line: - continue - lines.append(line) - if len(lines) > 4000: - lines = lines[-4000:] - log.write(line) - log.flush() - parsed = _event_payload(provider, line, state, state_file) - structured = parsed or structured - _terminate_worker_process(process, known_descendants) - remainder = process.stdout.read() - if remainder: - lines.append(remainder) - log.write(remainder) - for line in remainder.splitlines(): - parsed = _event_payload(provider, line, state, state_file) - structured = parsed or structured - if forced_failure: - log.write(forced_failure + "\n") - log.write(f"\n[{utc_now()}] {provider} exit={process.returncode}\n") - selector.close() - output = "".join(lines)[-100000:] - returncode = int(process.returncode if process.returncode is not None else 1) - return ProcessResult( - returncode=returncode, - output=output, - structured=structured, - capacity_failure=returncode != 0 and bool(CAPACITY_PATTERN.search(output)), - ) - - -def _process_record(pid: int) -> tuple[int, int, int] | None: - """Return one Linux process's parent, group, and start-time identity.""" - try: - raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") - fields = raw[raw.rfind(")") + 2 :].split() - return int(fields[1]), int(fields[2]), int(fields[19]) - except (IndexError, OSError, ValueError): - return None - - -def _descendant_processes(root_pid: int) -> dict[int, tuple[int, int]]: - """Snapshot descendants as pid -> (process group, start-time identity).""" - records: dict[int, tuple[int, int, int]] = {} - for entry in Path("/proc").iterdir(): - if not entry.name.isdigit(): - continue - pid = int(entry.name) - record = _process_record(pid) - if record is not None: - records[pid] = record - family = {root_pid} - changed = True - while changed: - changed = False - for pid, (parent, _, _) in records.items(): - if pid not in family and parent in family: - family.add(pid) - changed = True - return { - pid: (records[pid][1], records[pid][2]) - for pid in family - if pid != root_pid and pid in records - } - - -def _process_identity_matches(pid: int, start_time: int) -> bool: - record = _process_record(pid) - return record is not None and record[2] == start_time - - -def _signal_worker_tree( - root_pid: int, - descendants: dict[int, tuple[int, int]], - sig: signal.Signals, -) -> None: - """Signal captured descendants across terminal-created process groups.""" - groups = {root_pid} - for pid, (process_group, start_time) in descendants.items(): - if _process_identity_matches(pid, start_time): - groups.add(process_group) - for process_group in groups: - try: - os.killpg(process_group, sig) - except ProcessLookupError: - pass - for pid, (_, start_time) in descendants.items(): - if not _process_identity_matches(pid, start_time): - continue - try: - os.kill(pid, sig) - except ProcessLookupError: - pass - - -def _terminate_worker_process( - process: subprocess.Popen[str], - known_descendants: dict[int, tuple[int, int]] | None = None, -) -> None: - """Reap a worker and terminal descendants that created their own groups.""" - descendants = dict(known_descendants or {}) - descendants.update(_descendant_processes(process.pid)) - if process.poll() is None: - _signal_worker_tree(process.pid, descendants, signal.SIGTERM) - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - pass - - # Provider terminal tools create their own sessions, so killing only the - # native CLI's process group leaves test/build subprocesses orphaned. The - # identity check prevents a reused PID from targeting an unrelated worker. - _signal_worker_tree(process.pid, descendants, signal.SIGKILL) - if process.poll() is None: - process.wait(timeout=10) - - -def _base_env() -> dict[str, str]: - """Build a worker environment while preserving Vault-backed CLI homes.""" - env = os.environ.copy() - env.update( - { - "HOME": str(DATA_ROOT / "home"), - "CODEX_HOME": os.environ.get("CODEX_HOME", "/runtime-access/codex"), - "CLAUDE_CONFIG_DIR": os.environ.get( - "CLAUDE_CONFIG_DIR", "/runtime-access/claude" - ), - "GIT_TERMINAL_PROMPT": "0", - } - ) - env["PATH"] = f"{DATA_ROOT / 'tools/bin'}:/opt/coordinator:" + env.get("PATH", "") - return env - - -def _codex_command(route: Route, prompt: str, workspace: Path, state: dict[str, Any], result_file: Path) -> list[str]: - base = [ - str(CODEX_BIN), - "--dangerously-bypass-approvals-and-sandbox", - "--dangerously-bypass-hook-trust", - "exec", - ] - thread_id = str(state.get("codex_thread_id") or "") - common = [ - "--json", - "-m", - route.model, - "-c", - f'model_reasoning_effort="{route.effort}"', - "--output-schema", - str(RESULT_SCHEMA_PATH), - "-o", - str(result_file), - ] - if thread_id: - return [*base, "resume", *common, thread_id, prompt] - return [ - *base, - *common, - "-C", - str(workspace), - prompt, - ] - - -def _claude_command(route: Route, prompt: str, state: dict[str, Any], resume: bool) -> list[str]: - session_id = str(state["claude_session_id"]) - session = ["--resume", session_id] if resume else ["--session-id", session_id] - denied = [ - "Bash(git push --force *)", - "Bash(git reset --hard *)", - "Bash(git clean -f *)", - ] - return [ - str(CLAUDE_BIN), - "--dangerously-skip-permissions", - "--autocompact", - "auto", - "--settings", - str(CLAUDE_SETTINGS), - "--disallowedTools", - *denied, - "--model", - route.model, - "--effort", - route.effort, - "--output-format", - "stream-json", - "--verbose", - "--json-schema", - json.dumps(RESULT_SCHEMA, separators=(",", ":")), - *session, - "-p", - prompt, - ] - - -def run_provider( - route: Route, - prompt: str, - workspace: Path, - state: dict[str, Any], - state_file: Path, - log_path: Path, - heartbeat: Callable[[str], bool], - max_runtime: int, -) -> ProcessResult: - """Run or resume one provider session using its pinned structured CLI.""" - state.setdefault("attempts", []).append({"route": asdict(route), "started_at": utc_now()}) - if route.provider == "claude" and not state.get("claude_session_id"): - # The reservation is durable before Claude starts, closing the crash gap. - state["claude_session_id"] = str(uuid.uuid4()) - state["current_route"] = asdict(route) - state["updated_at"] = utc_now() - env = _base_env() - if route.provider == "codex": - result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1 - result_file = _result_path(state_file, state.get("run_id"), result_sequence) - while result_file.exists(): - result_sequence += 1 - result_file = _result_path( - state_file, - state.get("run_id"), - result_sequence, - ) - result_file.parent.mkdir(parents=True, exist_ok=True) - result_file.touch(mode=0o600, exist_ok=False) - state["result_sequence"] = result_sequence - state["current_result_file"] = str(result_file) - atomic_json(state_file, state) - command = _codex_command(route, prompt, workspace, state, result_file) - result = stream_process( - command, - provider="codex", - cwd=workspace, - env=env, - log_path=log_path, - state=state, - state_file=state_file, - heartbeat=heartbeat, - max_runtime=max_runtime, - ) - if ( - state.get("codex_thread_id") - and result.returncode != 0 - and NO_CODEX_THREAD in result.output.lower() - ): - state.pop("codex_thread_id", None) - result_sequence += 1 - result_file = _result_path( - state_file, - state.get("run_id"), - result_sequence, - ) - while result_file.exists(): - result_sequence += 1 - result_file = _result_path( - state_file, - state.get("run_id"), - result_sequence, - ) - result_file.touch(mode=0o600, exist_ok=False) - state["result_sequence"] = result_sequence - state["current_result_file"] = str(result_file) - atomic_json(state_file, state) - result = stream_process( - _codex_command(route, prompt, workspace, state, result_file), - provider="codex", - cwd=workspace, - env=env, - log_path=log_path, - state=state, - state_file=state_file, - heartbeat=heartbeat, - max_runtime=max_runtime, - ) - file_result = load_json(result_file) - if file_result.get("status") in cli_lane_goal.RESULT_STATUSES: - result.structured = file_result - try: - result_file.chmod(0o600) - except OSError: - pass - return result - - atomic_json(state_file, state) - resume = bool(state.get("claude_started")) - result = stream_process( - _claude_command(route, prompt, state, resume), - provider="claude", - cwd=workspace, - env=env, - log_path=log_path, - state=state, - state_file=state_file, - heartbeat=heartbeat, - max_runtime=max_runtime, - ) - if resume and result.returncode == 1 and NO_CLAUDE_SESSION in result.output: - result = stream_process( - _claude_command(route, prompt, state, False), - provider="claude", - cwd=workspace, - env=env, - log_path=log_path, - state=state, - state_file=state_file, - heartbeat=heartbeat, - max_runtime=max_runtime, - ) - elif not resume and result.returncode == 1 and CLAUDE_SESSION_COLLISION in result.output: - result = stream_process( - _claude_command(route, prompt, state, True), - provider="claude", - cwd=workspace, - env=env, - log_path=log_path, - state=state, - state_file=state_file, - heartbeat=heartbeat, - max_runtime=max_runtime, - ) - if result.returncode == 0 or NO_CLAUDE_SESSION not in result.output: - state["claude_started"] = True - atomic_json(state_file, state) - return result - - -def _task_value(task: Any, name: str, default: Any = None) -> Any: - return getattr(task, name, default) - - -def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str: - value = kanban_db.build_worker_context(conn, task_id) - if isinstance(value, str): - return value - return json.dumps(value, indent=2, default=str) - - -def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path: - # External coding lanes always receive their own linked worktree. A task - # that does not resolve to a Git repository is blocked rather than sharing - # a mutable checkout with another unattended worker. - # Git worktree creation updates common-repository metadata. Serialize that - # short materialization step while allowing the provider workers themselves - # to run concurrently in independent worktrees. - with WORKTREE_LOCK: - value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board) - kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name) - return Path(value).resolve() - - -def _board_call( - kanban_db: Any, - board: str, - operation: Callable[[Any], Any], -) -> Any: - """Run one bounded Kanban operation on a fresh, promptly closed connection.""" - last_storage_error: Exception | None = None - for attempt in range(KANBAN_STORAGE_ATTEMPTS): - conn = None - try: - with kanban_db.scoped_current_board(board): - conn = kanban_db.connect(board=board) - result = operation(conn) - BOARD_CORRUPTION_ERRORS.pop(board, None) - return result - except (OSError, sqlite3.Error) as error: - last_storage_error = error - _record_board_access_error(board, error) - if attempt + 1 < KANBAN_STORAGE_ATTEMPTS: - time.sleep(0.2 * (attempt + 1)) - finally: - if conn is not None: - conn.close() - assert last_storage_error is not None - raise last_storage_error - - -def _rename_noreplace( - source: str, - destination: str, - *, - source_dir: int, - destination_dir: int, -) -> None: - """Atomically rename a directory entry without replacing a collision.""" - renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None) - if renameat2 is None: - raise OSError(errno.ENOSYS, "renameat2 is unavailable") - renameat2.argtypes = ( - ctypes.c_int, - ctypes.c_char_p, - ctypes.c_int, - ctypes.c_char_p, - ctypes.c_uint, - ) - renameat2.restype = ctypes.c_int - result = renameat2( - source_dir, - ctypes.c_char_p(os.fsencode(source)), - destination_dir, - ctypes.c_char_p(os.fsencode(destination)), - 1, # RENAME_NOREPLACE - ) - if result != 0: - error = ctypes.get_errno() - raise OSError(error, os.strerror(error), source, destination) - - -def _retire_terminal_entry( - path: Path, - source_stat: os.stat_result, - *, - board_descriptor: int, - quarantine_descriptor: int, -) -> str: - """Retire only the inode previously inspected; preserve any replacement.""" - try: - current = os.stat( - path.name, - dir_fd=board_descriptor, - follow_symlinks=False, - ) - except FileNotFoundError: - return "missing" - if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino: - return "replacement" - identity = hashlib.sha256( - f"{path.name}\0{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode("utf-8") - ).hexdigest()[:32] - for sequence in range(32): - staging = f".retire.{path.name}.{identity}.{sequence}" - try: - _rename_noreplace( - path.name, - staging, - source_dir=board_descriptor, - destination_dir=quarantine_descriptor, - ) - except FileExistsError: - continue - break - else: - return "collision" - staged = os.stat( - staging, - dir_fd=quarantine_descriptor, - follow_symlinks=False, - ) - if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino: - os.unlink(staging, dir_fd=quarantine_descriptor) - os.fsync(quarantine_descriptor) - os.fsync(board_descriptor) - return "retired" - # The path changed after the pre-check but before the rename. Put the - # replacement back without overwriting an even newer entry. - try: - _rename_noreplace( - staging, - path.name, - source_dir=quarantine_descriptor, - destination_dir=board_descriptor, - ) - except FileExistsError: - return "replacement-staged" - os.fsync(quarantine_descriptor) - os.fsync(board_descriptor) - return "replacement" - - -def _quarantine_terminal(path: Path, identity: TerminalIdentity | None, reason: str) -> Path: - """Record bounded metadata, then retire only the inspected journal inode.""" - board_dir = path.parent - try: - board_stat = board_dir.stat(follow_symlinks=False) - except OSError: - board_stat = None - if board_stat is None or not stat.S_ISDIR(board_stat.st_mode): - print( - f"refused terminal journal path through unsafe board directory: {path}", - file=sys.stderr, - flush=True, - ) - return path - board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - board_flags |= getattr(os, "O_NOFOLLOW", 0) - file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - create_flags |= getattr(os, "O_NOFOLLOW", 0) - board_descriptor = None - quarantine_descriptor = None - source_descriptor = None - source_stat = None - destination = path - retirement = "not-attempted" - try: - board_descriptor = os.open(board_dir, board_flags) - source_stat = os.stat( - path.name, - dir_fd=board_descriptor, - follow_symlinks=False, - ) - safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1 - digest = None - hashed_bytes = 0 - hash_complete = False - if safe_source: - source_descriptor = os.open( - path.name, - file_flags, - dir_fd=board_descriptor, - ) - opened_stat = os.fstat(source_descriptor) - if ( - opened_stat.st_dev != source_stat.st_dev - or opened_stat.st_ino != source_stat.st_ino - or opened_stat.st_nlink != 1 - ): - raise OSError("terminal journal identity changed while opening") - # Keep this descriptor open through retirement. The staging entry - # must match the exact fstat-verified inode, not merely a path stat - # captured before a concurrent replacement. - source_stat = opened_stat - prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES) - hashed_bytes = len(prefix) - hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES - digest = hashlib.sha256(prefix).hexdigest() - source_kind = ( - "regular" - if safe_source - else "hardlink" - if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1 - else "non-regular" - ) - diagnostic = { - "hash_complete": hash_complete, - "hashed_bytes": hashed_bytes, - "original_name": path.name, - "reason": reason, - "sha256": digest, - "size": source_stat.st_size, - "source_kind": source_kind, - } - payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8") - fingerprint = ( - (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16] - ) - safe_reason = ( - re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid" - ) - try: - os.mkdir("quarantine", 0o700, dir_fd=board_descriptor) - except FileExistsError: - pass - quarantine_descriptor = os.open( - "quarantine", - board_flags, - dir_fd=board_descriptor, - ) - for sequence in range(32): - name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine" - try: - destination_descriptor = os.open( - name, - create_flags, - 0o600, - dir_fd=quarantine_descriptor, - ) - except FileExistsError: - continue - try: - with os.fdopen(destination_descriptor, "wb") as target: - target.write(payload) - target.flush() - os.fsync(target.fileno()) - target_stat = os.fstat(target.fileno()) - if ( - stat.S_IMODE(target_stat.st_mode) != 0o600 - or target_stat.st_nlink != 1 - ): - raise OSError("quarantine destination is not private and singly linked") - except Exception: - os.unlink(name, dir_fd=quarantine_descriptor) - raise - destination = board_dir / "quarantine" / name - break - else: - raise OSError("could not reserve a unique quarantine destination") - os.fsync(quarantine_descriptor) - retirement = _retire_terminal_entry( - path, - source_stat, - board_descriptor=board_descriptor, - quarantine_descriptor=quarantine_descriptor, - ) - except OSError as error: - # A bad quarantine target must not make an attacker-controlled pending - # path eligible for replay forever. The same identity-safe staging - # protocol works in the board directory when quarantine is unusable. - if board_descriptor is not None and source_stat is not None: - try: - retirement = _retire_terminal_entry( - path, - source_stat, - board_descriptor=board_descriptor, - quarantine_descriptor=( - quarantine_descriptor - if quarantine_descriptor is not None - else board_descriptor - ), - ) - except OSError: - retirement = "deferred" - print( - f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}", - file=sys.stderr, - flush=True, - ) - finally: - if source_descriptor is not None: - os.close(source_descriptor) - if quarantine_descriptor is not None: - os.close(quarantine_descriptor) - if board_descriptor is not None: - os.close(board_descriptor) - print( - "quarantined terminal journal " - f"{path.name}: {reason}; identity={identity or 'unparseable'}; " - f"retirement={retirement}", - file=sys.stderr, - flush=True, - ) - return destination - - -def _recover_exact_run( - kanban_db: Any, - identity: TerminalIdentity | None, - reason: str, -) -> bool: - """Make an exact external run retryable after journal recovery fails.""" - if identity is None: - return False - - def operation(conn: Any) -> bool: - task = kanban_db.get_task(conn, identity.task_id) - if task is None: - return False - if ( - str(_task_value(task, "status", "")) != "running" - or _task_value(task, "current_run_id", None) != identity.run_id - or not _external(task) - ): - return False - return bool( - kanban_db.reclaim_task( - conn, - identity.task_id, - reason=f"terminal journal recovery failed ({reason}); exact run may retry", - ) - ) - - try: - recovered = bool(_board_call(kanban_db, identity.board, operation)) - except Exception as error: - _record_board_access_error(identity.board, error) - return False - if recovered: - print( - f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} " - f"after terminal journal recovery: {reason}", - file=sys.stderr, - flush=True, - ) - return recovered - - -def _commit_terminal_file(path: Path, identity: TerminalIdentity, document: dict[str, Any]) -> Path: - """Persist committed state, then atomically retire a pending journal.""" - committed = _terminal_path( - state_path(identity.board, identity.task_id), - identity.run_id, - "committed", - ) - document["kanban_state"] = "committed" - document["committed_at"] = utc_now() - atomic_json(path, document) - os.replace(path, committed) - os.chmod(committed, 0o600, follow_symlinks=False) - _fsync_directory(committed.parent) - return committed - - -def _finalize_terminal_record( - kanban_db: Any, - path: Path, - _record: dict[str, Any] | None = None, -) -> str: - """Commit one exact-run terminal journal, or leave it safely pending.""" - identity = _terminal_identity(path) - if identity is None or identity.state != "pending": - return "invalid" - document = _load_terminal_json(path, identity) - if not _terminal_record_valid(document): - return "invalid" - if not _terminal_record_valid(document, identity): - return "foreign" - - def operation(conn: Any) -> str: - task = kanban_db.get_task(conn, identity.task_id) - if task is None: - return "stale" - status = str(_task_value(task, "status", "")) - if status == "done": - return ( - "committed" - if str(_task_value(task, "result", "") or "") == document["result"] - else "stale" - ) - current_run_id = _task_value(task, "current_run_id", None) - completion_guard: dict[str, int] - if status == "running" and current_run_id == identity.run_id: - completion_guard = {"expected_run_id": identity.run_id} - elif status in {"ready", "blocked"} and current_run_id is None: - # The journal may have survived an older post-persistence error - # path that ended its run. The patched DB verifies atomically that - # this is still the latest ended run before allowing completion. - completion_guard = {"replay_ended_run_id": identity.run_id} - else: - return "stale" - completed = kanban_db.complete_task( - conn, - identity.task_id, - result=document["result"], - summary=document["summary"], - metadata=document["metadata"], - **completion_guard, - ) - return "committed" if completed else "pending" - - outcome = str(_board_call(kanban_db, identity.board, operation)) - if outcome == "committed": - _commit_terminal_file(path, identity, document) - return outcome - - -def _replay_replacement_terminal( - kanban_db: Any, - path: Path, - identity: TerminalIdentity, -) -> tuple[bool, bool]: - """Replay a valid journal atomically substituted during quarantine.""" - replacement = _load_terminal_json(path, identity) - if not _terminal_record_valid(replacement, identity): - return False, False - try: - outcome = _finalize_terminal_record(kanban_db, path, replacement) - except Exception as error: - _record_board_access_error(identity.board, error) - return True, False - return True, outcome == "committed" - - -def recover_pending_finalizations() -> int: - """Replay accepted exact-run results before scheduling more provider work.""" - from hermes_cli import kanban_db - - recovered = 0 - for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")): - identity = _terminal_identity(path) - record = _load_terminal_json(path, identity) if identity is not None else {} - if identity is None: - _quarantine_terminal(path, None, "malformed-name") - continue - if not _terminal_record_valid(record): - _quarantine_terminal(path, identity, "malformed-payload") - replacement, committed = _replay_replacement_terminal( - kanban_db, path, identity - ) - if committed: - recovered += 1 - elif not replacement: - _recover_exact_run(kanban_db, identity, "malformed-payload") - continue - if not _terminal_record_valid(record, identity): - _quarantine_terminal(path, identity, "foreign-identity") - replacement, committed = _replay_replacement_terminal( - kanban_db, path, identity - ) - if committed: - recovered += 1 - elif not replacement: - _recover_exact_run(kanban_db, identity, "foreign-identity") - continue - try: - outcome = _finalize_terminal_record(kanban_db, path, record) - except Exception as error: - _record_board_access_error(identity.board, error) - continue - if outcome == "committed": - recovered += 1 - elif outcome in {"invalid", "foreign", "stale"}: - _quarantine_terminal(path, identity, outcome) - replacement, committed = _replay_replacement_terminal( - kanban_db, path, identity - ) - if committed: - recovered += 1 - elif not replacement: - _recover_exact_run(kanban_db, identity, outcome) - return recovered - - -def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool: - """Keep an exact run claimed while its accepted result awaits replay.""" - if not isinstance(run_id, int): - return False - path = _terminal_path(state_path(board, task_id), run_id, "pending") - try: - path.stat() - except FileNotFoundError: - return False - except OSError: - return False - identity = _terminal_identity(path) - if identity is None: - return False - record = _load_terminal_json(path, identity) - return _terminal_record_valid(record, identity) - - -def _artifact_gc_candidates(board_dir: Path) -> list[Path]: - """List only bounded artifact classes; pending journals are excluded.""" - candidates: list[Path] = [] - patterns = ( - "*.provider-*.result.json", - "*.candidate-*.json", - "*.terminal.committed.json", - ) - for pattern in patterns: - candidates.extend(board_dir.glob(pattern)) - quarantine = board_dir / "quarantine" - if quarantine.is_dir(): - candidates.extend(quarantine.glob("*.quarantine")) - return candidates - - -def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool: - """Remove only the exact artifact inode selected by retention.""" - directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - directory_flags |= getattr(os, "O_NOFOLLOW", 0) - descriptor = None - try: - descriptor = os.open(path.parent, directory_flags) - outcome = _retire_terminal_entry( - path, - observed, - board_descriptor=descriptor, - quarantine_descriptor=descriptor, - ) - return outcome == "retired" - except OSError: - return False - finally: - if descriptor is not None: - os.close(descriptor) - - -def gc_lane_artifacts( - *, - now: float | None = None, - max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS, - max_count: int = ARTIFACT_RETENTION_COUNT, - max_bytes: int = ARTIFACT_RETENTION_BYTES, -) -> int: - """Bound non-pending lane artifacts by age, count, and bytes per board.""" - current = time.time() if now is None else now - removed = 0 - for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else (): - if not board_dir.is_dir() or board_dir.is_symlink(): - continue - board_removed = 0 - entries: list[tuple[Path, os.stat_result]] = [] - for path in _artifact_gc_candidates(board_dir): - try: - file_stat = path.stat(follow_symlinks=False) - except OSError: - continue - if not stat.S_ISREG(file_stat.st_mode): - if _unlink_artifact_if_same(path, file_stat): - removed += 1 - board_removed += 1 - continue - if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds: - if _unlink_artifact_if_same(path, file_stat): - removed += 1 - board_removed += 1 - else: - entries.append((path, file_stat)) - retained_count = 0 - retained_bytes = 0 - for path, file_stat in sorted( - entries, key=lambda item: item[1].st_mtime, reverse=True - ): - exceeds_count = max_count >= 0 and retained_count >= max_count - exceeds_bytes = ( - max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes - ) - if exceeds_count or exceeds_bytes: - if _unlink_artifact_if_same(path, file_stat): - removed += 1 - board_removed += 1 - continue - retained_count += 1 - retained_bytes += file_stat.st_size - if board_removed: - _fsync_directory(board_dir) - return removed - - -def maybe_gc_lane_artifacts(*, now: float | None = None) -> int: - """Run artifact retention on a bounded cadence, not every dispatcher tick.""" - global LAST_ARTIFACT_GC - current = time.time() if now is None else now - if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS: - return 0 - LAST_ARTIFACT_GC = current - try: - return gc_lane_artifacts(now=current) - except OSError as error: - print( - f"lane artifact retention deferred: {type(error).__name__}: {error}", - file=sys.stderr, - flush=True, - ) - return 0 - - -def execute_claim(board: str, task_id: str) -> None: - """Execute one already-claimed task and commit its outcome to Kanban.""" - from hermes_cli import kanban_db - - run_id = None - with kanban_db.scoped_current_board(board): - conn = kanban_db.connect(board=board) - try: - task = kanban_db.get_task(conn, task_id) - if task is None: - return - run_id = _task_value(task, "current_run_id", None) - assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto") - state_file = state_path(board, task_id) - state = load_json(state_file) - state.update( - { - "board": board, - "task_id": task_id, - "run_id": run_id, - "assignee": assignee, - } - ) - atomic_json(state_file, state) - log_path = Path(kanban_db.worker_log_path(task_id, board=board)) - workspace = _resolve_workspace(kanban_db, conn, task, board) - kanban_db.set_workspace_path(conn, task_id, str(workspace)) - context = _task_context(kanban_db, conn, task_id) - except Exception as error: - failure_reason = ( - f"Direct CLI lane preparation failed: {type(error).__name__}: {error}" - ) - conn.close() - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.block_task( - fresh, - task_id, - reason=failure_reason, - kind="capability", - expected_run_id=run_id, - ), - ) - return - finally: - conn.close() - - def heartbeat(note: str) -> bool: - try: - return bool( - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.heartbeat_worker( - fresh, - task_id, - note=note, - expected_run_id=run_id, - ), - ) - ) - except (OSError, sqlite3.Error): - # The claim TTL is deliberately long. Keep the expensive - # provider process alive during a transient volume stall and - # retry on the next heartbeat instead of losing its work. - return True - - def comment(body: str) -> None: - try: - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.add_comment( - fresh, - task_id, - "cli-lane-runner", - body, - ), - ) - except (OSError, sqlite3.Error): - # Route state is also written to the durable lane-state file; - # a later heartbeat or terminal result remains authoritative. - pass - - try: - previous_route = state.get("current_route") - excluded_provider = ( - fresh_unavailable_provider() if assignee == "cli-auto" else None - ) - unavailable_provider = excluded_provider - route = select_route( - context, - assignee, - exclude_provider=excluded_provider, - exclude_reason="is unavailable according to fresh native health" - if excluded_provider - else None, - ) - if excluded_provider: - comment( - f"Provider health guard excluded {excluded_provider} before automatic routing.", - ) - comment( - f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}", - ) - resume_handoff = "" - if ( - isinstance(previous_route, dict) - and previous_route.get("provider") - and previous_route.get("provider") != route.provider - ): - try: - previous_output = log_path.read_text(encoding="utf-8")[-10000:] - except OSError: - previous_output = "Previous provider log was unavailable after restart." - resume_handoff = git_handoff(workspace, previous_output) - comment( - f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.", - ) - prompt = build_prompt(context, workspace, resume_handoff) - max_runtime = int( - _task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME - ) - goal_mode = bool(_task_value(task, "goal_mode", False)) - goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1)) - goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1) - deadline = time.monotonic() + max_runtime - while True: - state["goal_turn"] = goal_turn - atomic_json(state_file, state) - remaining = max(1, int(deadline - time.monotonic())) - result = run_provider( - route, - prompt, - workspace, - state, - state_file, - log_path, - heartbeat, - remaining, - ) - candidate_file = None - if result.structured: - candidate_file = _persist_candidate( - state, - state_file, - dict(result.structured), - route=route, - returncode=result.returncode, - goal_turn=goal_turn, - ) - if result.capacity_failure: - unavailable_provider = route.provider - retry_context = ( - context - + "\n\nRouting boundary: the first provider failed from capacity/authentication. " - + "Select the alternate hosted provider at an appropriate effort." - ) - alternate = "claude" if route.provider == "codex" else "codex" - fallback = select_route( - retry_context, - f"cli-{alternate}-{route.effort}", - ) - comment( - f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.", - ) - result = run_provider( - fallback, - build_prompt( - context, - workspace, - git_handoff(workspace, result.output), - ), - workspace, - state, - state_file, - log_path, - heartbeat, - max(1, int(deadline - time.monotonic())), - ) - route = fallback - if result.structured: - candidate_file = _persist_candidate( - state, - state_file, - dict(result.structured), - route=route, - returncode=result.returncode, - goal_turn=goal_turn, - ) - structured = result.structured - if structured: - structured = dict(structured) - structured["artifacts"] = workspace_artifacts( - workspace, - structured.get("artifacts"), - ) - metadata = { - "executor": "direct-cli-lane", - "provider": route.provider, - "model": route.model, - "effort": route.effort, - "classifier": route.classifier, - "state_file": str(state_file), - "codex_thread_id": state.get("codex_thread_id"), - "claude_session_id": state.get("claude_session_id"), - "goal_mode": goal_mode, - "goal_turn": goal_turn, - } - if candidate_file is not None: - metadata["candidate_file"] = str(candidate_file) - if structured: - for key in ( - "changed_files", - "tests_run", - "artifacts", - "findings", - "blockers", - ): - value = structured.get(key) - metadata[key] = value if isinstance(value, list) else [] - - completion_problem = None - if structured and result.returncode == 0: - completion_problem = cli_lane_goal.unfinished_result_reason(structured) - if ( - structured.get("status") == "completed" - and completion_problem is None - and goal_mode - ): - heartbeat("local goal-completion judge active") - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] - judge_context = context - if goal_turn > 1 or rejection_history: - judge_context += ( - "\n\nAuthoritative Hermes goal-controller evidence: " - f"current turn {goal_turn}/{goal_max_turns}; prior rejected " - f"reports: {json.dumps(rejection_history[-5:])}." - ) - accepted, judge_reason = cli_lane_goal.judge_goal_completion( - judge_context, - structured, - ) - metadata["goal_judge_reason"] = judge_reason - if not accepted: - completion_problem = f"local goal judge requested continuation: {judge_reason}" - - if ( - structured - and structured.get("status") == "completed" - and result.returncode == 0 - and completion_problem is None - ): - terminal_file = _terminal_path(state_file, run_id, "pending") - metadata["terminal_record"] = str( - _terminal_path(state_file, run_id, "committed") - ) - try: - terminal_file, terminal_record = _write_terminal_record( - state_file, - board=board, - task_id=task_id, - run_id=run_id, - structured=structured, - summary=str(structured.get("summary") or "Completed"), - metadata=metadata, - ) - outcome = _finalize_terminal_record( - kanban_db, - terminal_file, - terminal_record, - ) - except Exception as error: - identity = _terminal_identity(terminal_file) - replayable = bool( - identity is not None - and _terminal_record_valid( - _load_terminal_json(terminal_file, identity), - identity, - ) - ) - if not replayable: - _recover_exact_run( - kanban_db, - identity, - f"persistence raised {type(error).__name__}", - ) - raise TerminalFinalizationPending( - "accepted worker result remains outside the generic block path; " - f"terminal replayable={replayable}; persistence/finalization " - f"raised {type(error).__name__}" - ) from error - if outcome != "committed": - raise TerminalFinalizationPending( - "accepted worker result is durably journaled but " - f"Kanban finalization is {outcome}" - ) - break - - can_continue = ( - goal_mode - and completion_problem is not None - and goal_turn < goal_max_turns - and deadline - time.monotonic() > 30 - ) - if can_continue: - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] - state["goal_rejections"] = [ - *rejection_history[-4:], - completion_problem, - ] - goal_turn += 1 - comment( - f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}", - ) - escalation_context = ( - context - + "\n\nThe previous worker turn missed its completion quality mark: " - + completion_problem - + "\nSelect a route that can finish and verify the remaining work." - ) - excluded = unavailable_provider - if excluded is None and assignee == "cli-auto": - excluded = fresh_unavailable_provider() - next_route = select_route( - escalation_context, - assignee, - exclude_provider=excluded, - exclude_reason="is unavailable according to fresh native health" - if excluded - else None, - ) - comment( - f"Goal route {goal_turn}/{goal_max_turns}: " - f"{next_route.provider}/{next_route.model} at {next_route.effort}; " - f"classifier={next_route.classifier}; {next_route.reason}", - ) - handoff = ( - git_handoff(workspace, result.output) - if next_route.provider != route.provider - else "" - ) - route = next_route - prompt = build_prompt( - context, - workspace, - handoff - + "\nGoal-loop continuation: the previous final report was rejected because " - + completion_problem - + ". Reinspect live state, finish the outstanding work, and return new final evidence.", - ) - continue - - reason = completion_problem - if reason is None and structured: - blockers = structured.get("blockers", []) - reason = "; ".join(str(item) for item in blockers) - reason = reason or str(structured.get("summary") or "") - if reason is None: - reason = result.output[-4000:] - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.block_task( - fresh, - task_id, - reason=reason - or f"{route.provider} worker failed with exit {result.returncode}", - kind="transient" if result.capacity_failure else "capability", - expected_run_id=run_id, - ), - ) - break - except TerminalFinalizationPending as error: - comment(str(error)) - except Exception as error: - failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}" - _board_call( - kanban_db, - board, - lambda fresh: kanban_db.block_task( - fresh, - task_id, - reason=failure_reason, - kind="capability", - expected_run_id=run_id, - ), - ) - - -def _board_slug(board: Any) -> str: - if isinstance(board, dict): - return str(board.get("slug") or board.get("id") or "") - return str(getattr(board, "slug", None) or getattr(board, "id", None) or board) - - -def _external(task: Any) -> bool: - return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX) - - -def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None: - """Open one board without letting localized storage faults stop other lanes.""" - try: - return kanban_db.connect(board=board) - except Exception as error: - _record_board_access_error(board, error) - return None - - -def _record_board_access_error(board: str, error: Exception) -> None: - """Report one board access failure once while allowing other lanes to run.""" - failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access" - detail = f"{failure_kind} {type(error).__name__}: {error}" - if BOARD_CORRUPTION_ERRORS.get(board) == detail: - return - print( - f"temporarily skipping Kanban board {board!r}: {detail}", - file=sys.stderr, - flush=True, - ) - BOARD_CORRUPTION_ERRORS[board] = detail - - -def recover_orphans() -> None: - """Return external running tasks to ready after a runner/pod restart.""" - from hermes_cli import kanban_db - - recover_pending_finalizations() - try: - boards = kanban_db.list_boards(include_archived=False) - except Exception as error: - _record_board_access_error("board-registry", error) - return - for raw_board in boards: - board = _board_slug(raw_board) - if not board: - continue - with kanban_db.scoped_current_board(board): - conn = _connect_healthy_board(kanban_db, board) - if conn is None: - continue - try: - for task in kanban_db.list_tasks(conn): - if _external(task) and str(_task_value(task, "status", "")) == "running": - task_id = str(_task_value(task, "id")) - run_id = _task_value(task, "current_run_id", None) - if _has_pending_finalization(board, task_id, run_id): - continue - kanban_db.reclaim_task( - conn, - task_id, - reason="direct CLI lane restarted; provider session will resume", - ) - BOARD_CORRUPTION_ERRORS.pop(board, None) - except Exception as error: - _record_board_access_error(board, error) - finally: - conn.close() - - -def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: - """Atomically claim external ready tasks across all non-archived boards.""" - from hermes_cli import kanban_db - - claimed: list[tuple[str, str]] = [] - if limit <= 0: - return claimed - for raw_board in kanban_db.list_boards(include_archived=False): - board = _board_slug(raw_board) - if not board: - continue - with kanban_db.scoped_current_board(board): - conn = _connect_healthy_board(kanban_db, board) - if conn is None: - continue - try: - kanban_db.recompute_ready(conn) - tasks = kanban_db.list_tasks(conn) - BOARD_CORRUPTION_ERRORS.pop(board, None) - for task in tasks: - task_id = str(_task_value(task, "id", "")) - assignee = str(_task_value(task, "assignee", "") or "") - if task_id and not assignee and str(_task_value(task, "status", "")) == "ready": - if kanban_db.assign_task(conn, task_id, "cli-auto"): - task = kanban_db.get_task(conn, task_id) - assignee = "cli-auto" - if ( - not task_id - or (board, task_id) in active - or not assignee.startswith(EXTERNAL_PREFIX) - or str(_task_value(task, "status", "")) != "ready" - ): - continue - try: - result = kanban_db.claim_task( - conn, - task_id, - ttl_seconds=DEFAULT_CLAIM_TTL, - claimer="direct-cli-lane", - ) - except Exception: - continue - if result is not None: - claimed.append((board, task_id)) - if len(claimed) >= limit: - return claimed - except Exception as error: - _record_board_access_error(board, error) - finally: - conn.close() - return claimed - - -def main() -> int: - """Continuously bridge external Kanban lanes to provider CLIs.""" - RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) - atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644) - recover_orphans() - workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8)) - futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: - while True: - for future in list(futures): - if future.done(): - try: - future.result() - except Exception as error: - print(f"worker future failed: {error}", file=sys.stderr, flush=True) - del futures[future] - recover_pending_finalizations() - maybe_gc_lane_artifacts() - active = set(futures.values()) - try: - newly_claimed = claim_ready(active, workers - len(futures)) - BOARD_CORRUPTION_ERRORS.pop("board-registry", None) - except Exception as error: - _record_board_access_error("board-registry", error) - newly_claimed = [] - for board, task_id in newly_claimed: - future = pool.submit(execute_claim, board, task_id) - futures[future] = (board, task_id) - time.sleep(5) - return 0 +from cli_lane_execution import execute_claim +from cli_lane_files import ( + _candidate_path, + _fsync_directory, + _result_path, + _terminal_evidence_identity, + _terminal_identity, + _terminal_path, + atomic_json, + load_json, + state_path, +) +from cli_lane_finalization import ( + _discard_evidence, + _finalize_document_db, + _finalize_terminal_record, + _promote_prepared_evidence, + _recover_exact_run, + _resolve_pending_after_winner, + _retire_snapshot, + _retire_snapshot_after_db, +) +from cli_lane_prompt import ( + _event_payload, + _extract_json, + build_prompt, + git_handoff, + workspace_artifacts, +) +from cli_lane_provider import ( + _base_env, + _claude_command, + _codex_command, + _descendant_processes, + _process_identity_matches, + _process_record, + _signal_worker_tree, + _terminate_worker_process, + run_provider, + stream_process, +) +from cli_lane_quarantine import _quarantine_terminal +from cli_lane_records import ( + _load_small_json, + _load_terminal_json, + _open_small_json_snapshot, + _open_terminal_recovery_snapshot, + _open_terminal_snapshot, + _persist_candidate, + _read_bounded, + _terminal_record_valid, + _write_terminal_record, +) +from cli_lane_recovery import ( + _drain_retirement_staging, + _has_pending_finalization, + _recover_prepared_finalizations, + _recover_retirement_staging, + _restore_staged_terminal_authority, + _staged_terminal_authority, + _terminal_entry_absent, + recover_pending_finalizations, +) +from cli_lane_retention import ( + _artifact_gc_candidates, + _unlink_artifact_if_same, + gc_lane_artifacts, + maybe_gc_lane_artifacts, +) +from cli_lane_routing import fresh_unavailable_provider, parse_assignee, select_route if __name__ == "__main__": diff --git a/services/hermes/scripts/recover_cassandra_workers.py b/services/hermes/scripts/recover_cassandra_workers.py index ed0c12da..b0bb75c4 100755 --- a/services/hermes/scripts/recover_cassandra_workers.py +++ b/services/hermes/scripts/recover_cassandra_workers.py @@ -24,10 +24,12 @@ def recover_running_tasks(kanban_db: Any) -> list[str]: if str(getattr(task, "status", "")) != "running": continue task_id = str(getattr(task, "id", "")) - if not task_id or not kanban_db.reclaim_task( + run_id = getattr(task, "current_run_id", None) + if not task_id or not isinstance(run_id, int) or not kanban_db.reclaim_task( connection, task_id, reason=REASON, + expected_run_id=run_id, ): continue kanban_db.add_comment( diff --git a/testing/tests/test_hermes_agent_access.py b/testing/tests/test_hermes_agent_access.py new file mode 100644 index 00000000..d4138256 --- /dev/null +++ b/testing/tests/test_hermes_agent_access.py @@ -0,0 +1,253 @@ +"""Owner and Switchyard access-boundary contracts.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + yaml, +) + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert annotations[ + "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" + ] == "kv/data/atlas/hermes/developer-ssh" + assert annotations[ + "vault.hashicorp.com/agent-inject-secret-node-ssh-config" + ] == "kv/data/atlas/hermes/developer-ssh" + assert annotations[ + "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" + ] == "kv/data/atlas/hermes/developer-ssh" + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ + "resources" + ] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "cat /vault/secrets/node-ssh-public-key" in reconciler + assert "grep -qxF" in reconciler + assert "for user in atlas oceanus" in reconciler + assert "/host-etc/passwd" in reconciler + assert "chown \"${uid}:${gid}\"" in reconciler + host_passwd = next( + item for item in pod["volumes"] if item["name"] == "host-passwd" + ) + assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load( + (HERMES / "switchyard-deployment.yaml").read_text() + ) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next(item for item in pod["containers"] if item["name"] == container_name) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ + "name" + ] + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py new file mode 100644 index 00000000..f5bd5074 --- /dev/null +++ b/testing/tests/test_hermes_agent_layout.py @@ -0,0 +1,323 @@ +"""Agent deployment layout and runtime policy.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + HERMES, + Path, + _agent_deployment, + _services, + client_config, + json, + migration, + policy, + pytest, + yaml, +) + + +@pytest.mark.parametrize( + "command", + [ + "git push --force origin main", + "git reset --hard HEAD~1", + "git clean -fd", + ], +) +def test_claude_pretool_hook_blocks_hard_denies(command: str): + assert policy.denial_reason(command) + + +def test_claude_pretool_hook_allows_normal_engineering(): + assert policy.denial_reason("pytest -q testing/tests") is None + assert policy.denial_reason("git push origin feature/hermes") is None + assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None + assert policy.denial_reason("flux reconcile kustomization hermes") is None + assert policy.denial_reason("vault kv get kv/atlas/hermes") is None + + +def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): + state = tmp_path / ".claude.json" + settings = tmp_path / "settings.json" + state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") + settings.write_text( + json.dumps( + { + "theme": "dark", + "permissions": { + "deny": [ + "Bash(kubectl apply *)", + "Bash(flux reconcile *)", + "Bash(vault kv *)", + "Bash(custom-owner-rule *)", + ] + }, + } + ) + + "\n", + encoding="utf-8", + ) + + client_config.configure_claude_state(state) + client_config.configure_claude_settings(settings) + + state_value = json.loads(state.read_text()) + settings_value = json.loads(settings.read_text()) + assert state_value["promptQueueUseCount"] == 4 + assert state_value["bypassPermissionsModeAccepted"] is True + assert settings_value["theme"] == "dark" + assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] + assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] + assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] + assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] + assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] + hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] + assert "claude_command_policy.py" in hook["command"] + + +def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): + session = tmp_path / "home/.config/herdr/session.json" + session.parent.mkdir(parents=True) + session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") + binary = tmp_path / "tools/bin/herdr" + binary.parent.mkdir(parents=True) + binary.write_text("legacy", encoding="utf-8") + (tmp_path / "home/.claude").mkdir() + (tmp_path / "home/.codex").mkdir() + + archive = migration.archive_legacy_state(tmp_path) + + value = json.loads(archive.read_text()) + assert value["legacy_session"]["agents"][0]["session_id"] == "abc" + assert (tmp_path / "home/.claude").is_dir() + assert (tmp_path / "home/.codex").is_dir() + assert not binary.exists() + assert not (tmp_path / "home/.config/herdr").exists() + + +def test_agent_uses_one_native_kanban_control_plane(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/balanced", + "model": "atlas/auto/balanced", + } + assert config["kanban"]["dispatch_in_gateway"] is True + assert config["kanban"]["default_assignee"] == "cli-auto" + assert config["plugins"]["enabled"] == ["auto-router"] + + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + assert pod["enableServiceLinks"] is False + names = {item["name"] for item in pod["containers"]} + assert "cli-lane-runner" in names + assert "terminal" in names + assert not any("herdr" in name for name in names) + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert "herdr server" not in rendered + assert "herdr-dispatch" not in rendered + + +def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): + deployment = _agent_deployment() + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + lane = containers["cli-lane-runner"] + environment = {item["name"]: item["value"] for item in lane["env"]} + + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert lane["resources"] == { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "2", "memory": "6Gi"}, + } + + +def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): + """Placement correction: keep the agent off nodes that cannot hold it. + + titan-04 is cordoned after repeated kernel undervoltage and kubelet + failure, and titan-19 was probe/Longhorn unstable under worker load, so + both must join the existing hard exclusions. That leaves titan-05 as the + healthy candidate, which is tight enough on requested CPU that the main + container has to give back 50m to schedule there. + """ + pod = _agent_deployment()["spec"]["template"]["spec"] + hostnames = next( + item + for item in pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + if item["key"] == "kubernetes.io/hostname" + ) + + assert hostnames["operator"] == "NotIn" + assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + + hermes = next( + item for item in pod["containers"] if item["name"] == "hermes" + ) + assert hermes["resources"]["requests"]["cpu"] == "300m" + + +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + assert "dashboard" not in containers + hermes = containers["hermes"] + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" + assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" + assert hermes["command"] == ["/bin/sh", "-ec"] + startup = hermes["args"][0] + assert ". /opt/data/.env" in startup + assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_DASHBOARD"] == "1" + assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" + assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" + assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + assert hermes["securityContext"]["runAsUser"] == 0 + assert hermes["securityContext"]["runAsGroup"] == 0 + for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): + probe = hermes[probe_name] + assert probe["exec"]["command"] == [ + "curl", + "-fsS", + "http://127.0.0.1:9119/api/status", + ] + + terminal = containers["terminal"] + command = terminal["args"][0] + assert "--base-path /terminal" in command + assert "--check-origin" not in command + assert "/usr/bin/tmux new-session -A" in command + assert "--continue" in command + assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + patch_init = next( + item for item in pod["initContainers"] + if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("hermes", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts + + ingress_documents = [ + item + for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if item + ] + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] == "http://127.0.0.1:9119" + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_cli_lane_domain_modules_are_all_mounted_with_the_runner(): + manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text()) + coordinator = next( + item + for item in manifest["configMapGenerator"] + if item["name"] == "hermes-coordinator" + ) + mounted = {entry.split("=", 1)[0] for entry in coordinator["files"]} + expected = { + f"cli_lane_{domain}.py" + for domain in ( + "board", + "config", + "dispatch", + "evidence", + "execution", + "files", + "finalization", + "goal", + "prompt", + "provider", + "quarantine", + "records", + "recovery", + "retention", + "routing", + "runner", + ) + } + assert expected <= mounted + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = ( + HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" + ).read_text(encoding="utf-8") + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile diff --git a/testing/tests/test_hermes_agent_runtime_patches.py b/testing/tests/test_hermes_agent_runtime_patches.py new file mode 100644 index 00000000..b28ce798 --- /dev/null +++ b/testing/tests/test_hermes_agent_runtime_patches.py @@ -0,0 +1,329 @@ +"""Pinned operator tooling and runtime patch contracts.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + HERMES, + Path, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + json, + pytest, + ttyd_patch, + tui_gateway_patch, + yaml, +) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert ( + "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" + in script + ) + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] + if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert 'configured = 180.0' in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py new file mode 100644 index 00000000..cf23a2d0 --- /dev/null +++ b/testing/tests/test_hermes_agent_security.py @@ -0,0 +1,202 @@ +"""Agent authentication, networking, and Flux health contracts.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + _agent_deployment, + yaml, +) + + +def test_agent_image_runs_execution_safety_patch_and_regressions(): + dockerfiles = HERMES.parents[1] / "dockerfiles" + dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8") + dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text( + encoding="utf-8" + ) + + for name in ( + "patch-hermes-execution-safety.py", + "hermes-execution-safety-regression.py", + "hermes_execution_patch_support.py", + "patch_hermes_run_safety.py", + "patch_hermes_decomposition_safety.py", + "hermes_execution_regression_support.py", + "hermes_run_safety_regression.py", + "hermes_decomposition_safety_regression.py", + ): + assert f"COPY dockerfiles/{name}" in dockerfile + assert f"!dockerfiles/{name}" in dockerignore + assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in ( + dockerfile + ) + assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in ( + dockerfile + ) + assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile + assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore + assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] + for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] + for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = { + (item["kind"], item["name"]) + for item in flux["spec"]["healthChecks"] + } + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "traefik" + } + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [ + { + "podSelector": { + "matchLabels": {"app": "hermes-chat-tenant"} + } + } + ], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [ + { + "podSelector": { + "matchLabels": {"app": "hermes-switchyard"} + } + } + ], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [{"protocol": "TCP", "port": 9010}], + }, + ] + assert isolation["spec"]["egress"] == [{}] + + +def test_owner_agent_has_cluster_admin_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-owner" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = HERMES / "agent-rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "cluster-admin", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] diff --git a/testing/tests/test_hermes_chat_codex_broker.py b/testing/tests/test_hermes_chat_codex_broker.py new file mode 100644 index 00000000..6ff10a23 --- /dev/null +++ b/testing/tests/test_hermes_chat_codex_broker.py @@ -0,0 +1,75 @@ +"""Hermes chat codex broker contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + ModuleType, + Path, + _load_broker_module, + base64, + json, + sys, + time, +) + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = base64.urlsafe_b64encode( + json.dumps({"exp": expires_at}).encode() + ).decode().rstrip("=") + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 diff --git a/testing/tests/test_hermes_chat_config.py b/testing/tests/test_hermes_chat_config.py new file mode 100644 index 00000000..7dd0e21c --- /dev/null +++ b/testing/tests/test_hermes_chat_config.py @@ -0,0 +1,273 @@ +"""Hermes chat config contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, + yaml, +) + + +def test_chat_config_enables_real_research_compute_and_delegation(): + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + + # AUTO leaves effort unset so the target chosen by Switchyard owns it. + assert "reasoning_effort" not in config["agent"] + assert config["model"]["model"] == "atlas/auto/fast" + assert config["web"] == { + "backend": "ddgs", + "search_backend": "ddgs", + "extract_backend": "public-extract", + } + assert config["delegation"]["max_concurrent_children"] == 2 + assert config["delegation"]["max_iterations"] == 80 + assert config["agent"]["max_turns"] == 120 + assert config["tool_loop_guardrails"]["hard_stop_enabled"] is True + for platform in ("cli", "api_server"): + toolsets = config["platform_toolsets"][platform] + assert "delegation" in toolsets + assert "browser" in toolsets + assert "python_sandbox" in toolsets + assert "vision" in toolsets + assert "web" in toolsets + assert "image_gen" in toolsets + assert "terminal" not in toolsets + assert "code_execution" not in toolsets + +def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): + configmap = _documents(HERMES / "agent-configmap.yaml")[0] + instructions = configmap["data"]["AGENTS.md"] + soul = configmap["data"]["SOUL.md"] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + + assert ( + "Only the foreground durable worker owns its Kanban task lifecycle" + in instructions + ) + assert "Never unblock and then complete" in instructions + assert "gateway dispatcher can claim the transient" in instructions + for platform in ("cli", "api_server"): + assert "kanban" in config["platform_toolsets"][platform] + assert "must never complete, block, unblock, reclaim" in instructions + assert "task's final structured result itself" in instructions + assert "Atlas organization has private visibility" in instructions + assert "may be public or private" in instructions + assert "do not infer a\nrepository's visibility" in instructions + assert "already supplied through `GIT_ASKPASS`" in instructions + assert "Never call `kanban_show` without a known, non-empty task ID" in instructions + assert "bounded ad-hoc inspection and acceptance checks may" in instructions + assert "load implementation or TDD skills" in instructions + assert "Never call `kanban_show` without\na known, non-empty task ID" in soul + assert "must load a skill only when its workflow\nmaterially applies" in soul + assert "runtime-only `GIT_ASKPASS`" in soul + assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions + assert "Never load or follow a GitHub/`gh`" in instructions + assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "JENKINS_BASE_URL" in instructions + assert "Do not\nuse `git reset --hard`" in instructions + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert 'git config --global user.name "Hermes Agent"' in rendered + assert 'git config --global user.email "hermes@bstein.dev"' in rendered + +def test_agent_image_completes_parked_kanban_tasks_atomically(): + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() + + assert ( + "AND status IN ('running', 'ready', 'blocked', 'scheduled')" + in dockerfile + ) + assert ( + "Atomically mark running, ready, blocked, or scheduled tasks done" + in dockerfile + ) + assert "kind IN ('created', 'blocked', 'unblocked')" in dockerfile + assert "payload.get(\"status\") == \"blocked\"" in dockerfile + assert "hermes-kanban-blocked-regression.py" in dockerfile + +def test_sandbox_shares_only_the_tenant_workspace_without_credentials(): + sandbox_docs = _documents(HERMES / "chat-sandbox.yaml") + deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"] + assert len(deployments) == 8 + for ordinal, deployment in enumerate(deployments): + pod_spec = deployment["spec"]["template"]["spec"] + container = pod_spec["containers"][0] + assert pod_spec["automountServiceAccountToken"] is False + assert container["securityContext"]["readOnlyRootFilesystem"] is True + assert container["securityContext"]["runAsNonRoot"] is True + assert container["securityContext"]["runAsGroup"] == 10000 + assert not container.get("env") + assert {mount["mountPath"] for mount in container["volumeMounts"]} == { + "/tmp", + "/workspace", + "/opt/data/workspace", + } + workspace_volume = next( + item for item in pod_spec["volumes"] if item["name"] == "workspace" + ) + assert workspace_volume["persistentVolumeClaim"]["claimName"] == ( + f"workspace-hermes-chat-tenant-{ordinal}" + ) + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + templates = statefulset["spec"]["volumeClaimTemplates"] + workspace = next(item for item in templates if item["metadata"]["name"] == "workspace") + assert workspace["spec"]["resources"]["requests"]["storage"] == "10Gi" + assert workspace["spec"]["accessModes"] == ["ReadWriteMany"] + + pod_spec = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") + startup = hermes["args"][0] + assert "hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox" in startup + assert any( + mount["name"] == "workspace" and mount["mountPath"] == "/opt/data/workspace" + for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + deny = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-chat-sandbox-deny" + ) + assert deny["spec"]["ingress"] == [] + assert deny["spec"]["egress"] == [] + for ordinal in range(4): + policy = next( + item + for item in policies + if item["metadata"]["name"] == f"hermes-chat-sandbox-tenant-{ordinal}" + ) + assert policy["spec"]["podSelector"]["matchLabels"][ + "ai.bstein.dev/tenant-ordinal" + ] == str(ordinal) + source = policy["spec"]["ingress"][0]["from"][0]["podSelector"][ + "matchLabels" + ] + assert source["statefulset.kubernetes.io/pod-name"] == ( + f"hermes-chat-tenant-{ordinal}" + ) + +def test_gateway_image_honors_ui_model_and_caps_reasoning(): + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() + assert ( + "FROM nousresearch/hermes-agent@sha256:" + "47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08" + ) in dockerfile + assert "_resolve_request_route" in dockerfile + assert 'allowed_providers = {"atlas-switchyard"}' in dockerfile + assert 'reasoning_effort=body.get("reasoning_effort")' in dockerfile + assert 'reasoning_config = {"enabled": True, "effort": "xhigh"}' in dockerfile + assert "ddgs==9.14.4" in dockerfile + assert "specific not in _LEGACY_WEB_BACKENDS" in dockerfile + assert '"pre_internal_route"' in dockerfile + assert "pre_internal_route hook failed" in dockerfile + assert '"pre_subagent_route"' in dockerfile + assert "pre_subagent_route hook failed" in dockerfile + assert "agent._hermes_explicit_model_pick = bool" in dockerfile + assert "agent._hermes_oneshot = True" in dockerfile + assert 'getattr(parent_agent, "_hermes_oneshot", False)' in dockerfile + assert 'var != "CLAUDE_CODE_OAUTH_TOKEN"' in dockerfile + assert '"source": "managed_claude_code_subscription"' in dockerfile + assert '"name": "Claude Code subscription (Switchyard)"' in dockerfile + assert "Anthropic API / direct OAuth (not used by Switchyard)" in dockerfile + assert "scheduleTerminalPaint" in dockerfile + assert "sessionTree.childrenByParent" in dockerfile + assert "SessionActivityPanel" in dockerfile + assert "include_children=${includeChildren}" in dockerfile + assert "_dashboard_session_is_active" in dockerfile + assert 'searchParams.get("lineage_root") || resumeParam' in dockerfile + assert "followActiveLineage" in dockerfile + assert 'next.delete("lineage_root")' in dockerfile + activity = ( + ROOT / "dockerfiles" / "hermes-session-activity-panel.tsx" + ).read_text() + assert "Live worker activity" in activity + assert "api.getSessionMessages" in activity + assert "aria-live=\"polite\"" in activity + assert "role=\"log\"" in activity + assert "aria-busy={running && messages.length === 0}" in activity + assert 'detail.source !== "api_server"' not in activity + assert "Poll-backed transcript; terminal rendering is not required." in activity + assert "DEFAULT_VISIBLE_MESSAGES = 250" in activity + assert "current + DEFAULT_VISIBLE_MESSAGES" in activity + assert 'Show up to{" "}' in activity + assert "Math.min(" in activity + assert "visibleMessages.map" in activity + assert "response.total_messages" in activity + assert "visibleLimit)," in activity + assert "function displayText(value: unknown)" in activity + assert "open && (" in activity + assert 'document.getElementById("hermes-resume-bootstrap")?.remove()' in activity + assert 'id="hermes-resume-bootstrap"' in dockerfile + assert 'role="status"' in dockerfile + assert 'aria-busy="true"' in dockerfile + assert "Opening Hermes worker activity…" in dockerfile + assert 'params.has("resume")' in dockerfile + assert "Open Telegram setup" in dockerfile + +def test_telegram_router_local_build_defaults_to_cluster_architecture(): + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-chat-router" + ).read_text(encoding="utf-8") + + assert "ARG TARGETOS=linux" in dockerfile + assert "ARG TARGETARCH=arm64" in dockerfile + assert "GOOS=${TARGETOS} GOARCH=${TARGETARCH}" in dockerfile + +def test_chat_oauth_allows_stale_service_worker_retirement(): + documents = _documents(HERMES / "oauth2-proxy.yaml") + deployment = next( + document + for document in documents + if document["kind"] == "Deployment" + and document["metadata"]["name"] == "oauth2-proxy-hermes-chat" + ) + args = deployment["spec"]["template"]["spec"]["containers"][0]["args"] + + assert "--skip-auth-route=GET=^/sw[.]js([?].*)?$" in args + assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in args + + template = (HERMES / "oauth2-proxy-templates" / "error.html").read_text() + assert 'http-equiv="refresh"' not in template + assert "Unable to find a valid CSRF token" in template + assert "expired or was already used" in template + assert "sessionStorage" in template + assert "Automatic recovery paused" in template + assert "/start?rd=/" in template + container = deployment["spec"]["template"]["spec"]["containers"][0] + assert "v7.15.3@sha256:10a1165743a192e" in container["image"] + assert "--cookie-csrf-per-request=true" in args + assert "--cookie-csrf-per-request-limit=8" in args + assert "--trusted-proxy-ip=10.42.0.0/16" in args + assert "--api-route=^/api/" in args + assert "--api-route=^/health$" in args + assert "--cookie-expire=168h" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_pod = agent["spec"]["template"]["spec"] + agent_oauth = next( + container for container in agent_pod["containers"] + if container["name"] == "oauth2-proxy" + ) + assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in agent_oauth["args"] + assert { + "name": "oauth-templates", + "mountPath": "/etc/oauth2-proxy/templates", + "readOnly": True, + } in agent_oauth["volumeMounts"] + agent_template_volume = next( + volume for volume in agent_pod["volumes"] + if volume["name"] == "oauth-templates" + ) + assert agent_template_volume["configMap"]["name"] == ( + "hermes-chat-oauth-templates" + ) diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py new file mode 100644 index 00000000..19ad8415 --- /dev/null +++ b/testing/tests/test_hermes_chat_images.py @@ -0,0 +1,284 @@ +"""Hermes chat images contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + Path, + SimpleNamespace, + _documents, + importlib, + sys, + time, + yaml, +) + + +def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): + """Follow-up edits resolve the source server-side and keep tool JSON small.""" + provider_module = SimpleNamespace( + DEFAULT_ASPECT_RATIO="square", + ImageGenProvider=object, + error_response=lambda **value: value, + normalize_reference_images=lambda value: value, + resolve_aspect_ratio=lambda value: value, + save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", + success_response=lambda **value: value, + ) + monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) + spec = importlib.util.spec_from_file_location( + "hermes_image_plugin", + HERMES / "plugins" / "image-gen-broker" / "__init__.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + image_dir = tmp_path / "cache" / "images" + image_dir.mkdir(parents=True) + older = image_dir / "atlas_flux-old.png" + newest = image_dir / "atlas_gpt-image-new.png" + older.write_bytes(b"older") + newest.write_bytes(b"newest") + older.touch() + time.sleep(0.001) + newest.touch() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + calls = [] + monkeypatch.setattr( + module, + "_handle_image_generate", + lambda args, route: calls.append((args, route)) or "ok", + ) + assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" + assert calls == [ + ( + { + "prompt": "make it a clown", + "image_url": str(newest.resolve()), + }, + "hosted", + ) + ] + +def test_chat_reasoning_uses_switchyard_without_owner_credentials(): + """Family pods use AUTO/manual routes without mounting owner credentials.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/fast", + "model": "atlas/auto/fast", + } + assert config["providers"]["atlas-switchyard"] == { + "name": "Automatic Router", + "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", + "api_key": "atlas-switchyard", + "default_model": "atlas/auto/fast", + "transport": "chat_completions", + } + assert config["platforms"]["api_server"]["extra"]["model_routes"] == { + route: {"provider": "atlas-switchyard", "model": route} + for route in [ + "atlas/auto/fast", + "atlas/auto/balanced", + "atlas/auto/deep", + "atlas/auto/maximum", + "atlas/manual/codex/luna", + "atlas/manual/codex/terra", + "atlas/manual/codex/sol", + "atlas/manual/claude/haiku", + "atlas/manual/claude/fable", + "atlas/manual/claude/sonnet", + "atlas/manual/claude/opus", + "atlas/manual/local/qwen-14b", + ] + } + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "codex-broker") + assert broker["ports"] == [ + {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { + "PYTHONPATH": "/opt/hermes", + "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", + "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", + }.items() + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-codex-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + assert service["spec"]["ports"] == [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP", + } + ] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + assert statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] == "20260816-telegram-topics" + pod_spec = statefulset["spec"]["template"]["spec"] + patch_init = next( + item for item in pod_spec["initContainers"] + if item["name"] == "patch-stream-recovery" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/conversation_loop.py", + "/patched/conversation_loop.py", + ] + hermes = next( + item + for item in pod_spec["containers"] + if item["name"] == "hermes" + ) + assert { + "name": "stream-recovery-patch", + "mountPath": "/opt/hermes/agent/conversation_loop.py", + "subPath": "conversation_loop.py", + } in hermes["volumeMounts"] + api_session_init = next( + item for item in pod_spec["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + assert "patch_api_server_sessions.py" in api_session_init["args"][0] + assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] + assert { + "name": "api-server-patch", + "mountPath": "/opt/hermes/gateway/platforms/api_server.py", + "subPath": "api_server.py", + } in hermes["volumeMounts"] + assert any( + volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] + ) + assert not any( + mount["mountPath"].endswith("/.codex") + for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + +def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + +def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) is None + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) diff --git a/testing/tests/test_hermes_chat_local_compute.py b/testing/tests/test_hermes_chat_local_compute.py new file mode 100644 index 00000000..8ce39358 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_compute.py @@ -0,0 +1,295 @@ +"""Hermes chat local compute contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + Path, + ROOT, + SimpleNamespace, + _documents, + importlib, + json, + tomllib, +) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ + "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" + ] + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] + / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert "Repeated quality misses require xhigh" in routes[ + "worker_auto_maximum" + ]["prompt"] + assert any( + target.startswith("local_") + for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ( + "ollama-models-titan20" + ) + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + '\n'.join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert 'OFFLOAD_MODE = os.environ.get(' in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item for item in container["volumeMounts"] if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py new file mode 100644 index 00000000..1db93094 --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -0,0 +1,192 @@ +"""Hermes chat provider auth contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + Path, + ROOT, + _documents, + _load_broker_module, + importlib, + json, +) + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert 'detail = tool_name if event_type in {' in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert 'heartbeats.get(session_id, 0.0)' in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" + assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py deleted file mode 100644 index 818f7679..00000000 --- a/testing/tests/test_hermes_chat_quality.py +++ /dev/null @@ -1,2242 +0,0 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" - -from __future__ import annotations - -import base64 -import importlib.util -import json -import sqlite3 -import sys -import time -import tomllib -from pathlib import Path -from types import ModuleType, SimpleNamespace - -import pytest -import yaml - - -ROOT = Path(__file__).parents[2] -HERMES = ROOT / "services" / "hermes" -VAULT = ROOT / "services" / "vault" - - -def _documents(path: Path) -> list[dict]: - return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] - - -def _load_broker_module(name: str, filename: str, monkeypatch): - """Load one broker with its mounted routing-catalog dependency.""" - catalog_path = HERMES / "scripts" / "routing_catalog.py" - catalog_spec = importlib.util.spec_from_file_location( - "routing_catalog", catalog_path - ) - assert catalog_spec and catalog_spec.loader - catalog = importlib.util.module_from_spec(catalog_spec) - catalog_spec.loader.exec_module(catalog) - monkeypatch.setitem(sys.modules, "routing_catalog", catalog) - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - - broker_path = HERMES / "scripts" / filename - spec = importlib.util.spec_from_file_location(name, broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_chat_config_enables_real_research_compute_and_delegation(): - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - - # AUTO leaves effort unset so the target chosen by Switchyard owns it. - assert "reasoning_effort" not in config["agent"] - assert config["model"]["model"] == "atlas/auto/fast" - assert config["web"] == { - "backend": "ddgs", - "search_backend": "ddgs", - "extract_backend": "public-extract", - } - assert config["delegation"]["max_concurrent_children"] == 2 - assert config["delegation"]["max_iterations"] == 80 - assert config["agent"]["max_turns"] == 120 - assert config["tool_loop_guardrails"]["hard_stop_enabled"] is True - for platform in ("cli", "api_server"): - toolsets = config["platform_toolsets"][platform] - assert "delegation" in toolsets - assert "browser" in toolsets - assert "python_sandbox" in toolsets - assert "vision" in toolsets - assert "web" in toolsets - assert "image_gen" in toolsets - assert "terminal" not in toolsets - assert "code_execution" not in toolsets - - -def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): - configmap = _documents(HERMES / "agent-configmap.yaml")[0] - instructions = configmap["data"]["AGENTS.md"] - soul = configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - - assert ( - "Only the foreground durable worker owns its Kanban task lifecycle" - in instructions - ) - assert "Never unblock and then complete" in instructions - assert "gateway dispatcher can claim the transient" in instructions - for platform in ("cli", "api_server"): - assert "kanban" in config["platform_toolsets"][platform] - assert "must never complete, block, unblock, reclaim" in instructions - assert "task's final structured result itself" in instructions - assert "Atlas organization has private visibility" in instructions - assert "may be public or private" in instructions - assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions - assert "Never call `kanban_show` without a known, non-empty task ID" in instructions - assert "bounded ad-hoc inspection and acceptance checks may" in instructions - assert "load implementation or TDD skills" in instructions - assert "Never call `kanban_show` without\na known, non-empty task ID" in soul - assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions - assert "JENKINS_BASE_URL" in instructions - assert "Do not\nuse `git reset --hard`" in instructions - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert 'git config --global user.name "Hermes Agent"' in rendered - assert 'git config --global user.email "hermes@bstein.dev"' in rendered - - -def test_agent_image_completes_parked_kanban_tasks_atomically(): - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() - - assert ( - "AND status IN ('running', 'ready', 'blocked', 'scheduled')" - in dockerfile - ) - assert ( - "Atomically mark running, ready, blocked, or scheduled tasks done" - in dockerfile - ) - assert "kind IN ('created', 'blocked', 'unblocked')" in dockerfile - assert "payload.get(\"status\") == \"blocked\"" in dockerfile - assert "hermes-kanban-blocked-regression.py" in dockerfile - - -def test_sandbox_shares_only_the_tenant_workspace_without_credentials(): - sandbox_docs = _documents(HERMES / "chat-sandbox.yaml") - deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"] - assert len(deployments) == 8 - for ordinal, deployment in enumerate(deployments): - pod_spec = deployment["spec"]["template"]["spec"] - container = pod_spec["containers"][0] - assert pod_spec["automountServiceAccountToken"] is False - assert container["securityContext"]["readOnlyRootFilesystem"] is True - assert container["securityContext"]["runAsNonRoot"] is True - assert container["securityContext"]["runAsGroup"] == 10000 - assert not container.get("env") - assert {mount["mountPath"] for mount in container["volumeMounts"]} == { - "/tmp", - "/workspace", - "/opt/data/workspace", - } - workspace_volume = next( - item for item in pod_spec["volumes"] if item["name"] == "workspace" - ) - assert workspace_volume["persistentVolumeClaim"]["claimName"] == ( - f"workspace-hermes-chat-tenant-{ordinal}" - ) - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - templates = statefulset["spec"]["volumeClaimTemplates"] - workspace = next(item for item in templates if item["metadata"]["name"] == "workspace") - assert workspace["spec"]["resources"]["requests"]["storage"] == "10Gi" - assert workspace["spec"]["accessModes"] == ["ReadWriteMany"] - - pod_spec = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") - startup = hermes["args"][0] - assert "hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox" in startup - assert any( - mount["name"] == "workspace" and mount["mountPath"] == "/opt/data/workspace" - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - deny = next( - item - for item in policies - if item["metadata"]["name"] == "hermes-chat-sandbox-deny" - ) - assert deny["spec"]["ingress"] == [] - assert deny["spec"]["egress"] == [] - for ordinal in range(4): - policy = next( - item - for item in policies - if item["metadata"]["name"] == f"hermes-chat-sandbox-tenant-{ordinal}" - ) - assert policy["spec"]["podSelector"]["matchLabels"][ - "ai.bstein.dev/tenant-ordinal" - ] == str(ordinal) - source = policy["spec"]["ingress"][0]["from"][0]["podSelector"][ - "matchLabels" - ] - assert source["statefulset.kubernetes.io/pod-name"] == ( - f"hermes-chat-tenant-{ordinal}" - ) - - -def test_gateway_image_honors_ui_model_and_caps_reasoning(): - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() - assert ( - "FROM nousresearch/hermes-agent@sha256:" - "47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08" - ) in dockerfile - assert "_resolve_request_route" in dockerfile - assert 'allowed_providers = {"atlas-switchyard"}' in dockerfile - assert 'reasoning_effort=body.get("reasoning_effort")' in dockerfile - assert 'reasoning_config = {"enabled": True, "effort": "xhigh"}' in dockerfile - assert "ddgs==9.14.4" in dockerfile - assert "specific not in _LEGACY_WEB_BACKENDS" in dockerfile - assert '"pre_internal_route"' in dockerfile - assert "pre_internal_route hook failed" in dockerfile - assert '"pre_subagent_route"' in dockerfile - assert "pre_subagent_route hook failed" in dockerfile - assert "agent._hermes_explicit_model_pick = bool" in dockerfile - assert "agent._hermes_oneshot = True" in dockerfile - assert 'getattr(parent_agent, "_hermes_oneshot", False)' in dockerfile - assert 'var != "CLAUDE_CODE_OAUTH_TOKEN"' in dockerfile - assert '"source": "managed_claude_code_subscription"' in dockerfile - assert '"name": "Claude Code subscription (Switchyard)"' in dockerfile - assert "Anthropic API / direct OAuth (not used by Switchyard)" in dockerfile - assert "scheduleTerminalPaint" in dockerfile - assert "sessionTree.childrenByParent" in dockerfile - assert "SessionActivityPanel" in dockerfile - assert "include_children=${includeChildren}" in dockerfile - assert "_dashboard_session_is_active" in dockerfile - assert 'searchParams.get("lineage_root") || resumeParam' in dockerfile - assert "followActiveLineage" in dockerfile - assert 'next.delete("lineage_root")' in dockerfile - activity = ( - ROOT / "dockerfiles" / "hermes-session-activity-panel.tsx" - ).read_text() - assert "Live worker activity" in activity - assert "api.getSessionMessages" in activity - assert "aria-live=\"polite\"" in activity - assert "role=\"log\"" in activity - assert "aria-busy={running && messages.length === 0}" in activity - assert 'detail.source !== "api_server"' not in activity - assert "Poll-backed transcript; terminal rendering is not required." in activity - assert "DEFAULT_VISIBLE_MESSAGES = 250" in activity - assert "current + DEFAULT_VISIBLE_MESSAGES" in activity - assert 'Show up to{" "}' in activity - assert "Math.min(" in activity - assert "visibleMessages.map" in activity - assert "response.total_messages" in activity - assert "visibleLimit)," in activity - assert "function displayText(value: unknown)" in activity - assert "open && (" in activity - assert 'document.getElementById("hermes-resume-bootstrap")?.remove()' in activity - assert 'id="hermes-resume-bootstrap"' in dockerfile - assert 'role="status"' in dockerfile - assert 'aria-busy="true"' in dockerfile - assert "Opening Hermes worker activity…" in dockerfile - assert 'params.has("resume")' in dockerfile - assert "Open Telegram setup" in dockerfile - - -def test_telegram_router_local_build_defaults_to_cluster_architecture(): - dockerfile = ( - ROOT / "dockerfiles" / "Dockerfile.hermes-chat-router" - ).read_text(encoding="utf-8") - - assert "ARG TARGETOS=linux" in dockerfile - assert "ARG TARGETARCH=arm64" in dockerfile - assert "GOOS=${TARGETOS} GOARCH=${TARGETARCH}" in dockerfile - - -def test_chat_oauth_allows_stale_service_worker_retirement(): - documents = _documents(HERMES / "oauth2-proxy.yaml") - deployment = next( - document - for document in documents - if document["kind"] == "Deployment" - and document["metadata"]["name"] == "oauth2-proxy-hermes-chat" - ) - args = deployment["spec"]["template"]["spec"]["containers"][0]["args"] - - assert "--skip-auth-route=GET=^/sw[.]js([?].*)?$" in args - assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in args - - template = (HERMES / "oauth2-proxy-templates" / "error.html").read_text() - assert 'http-equiv="refresh"' not in template - assert "Unable to find a valid CSRF token" in template - assert "expired or was already used" in template - assert "sessionStorage" in template - assert "Automatic recovery paused" in template - assert "/start?rd=/" in template - container = deployment["spec"]["template"]["spec"]["containers"][0] - assert "v7.15.3@sha256:10a1165743a192e" in container["image"] - assert "--cookie-csrf-per-request=true" in args - assert "--cookie-csrf-per-request-limit=8" in args - assert "--trusted-proxy-ip=10.42.0.0/16" in args - assert "--api-route=^/api/" in args - assert "--api-route=^/health$" in args - assert "--cookie-expire=168h" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_pod = agent["spec"]["template"]["spec"] - agent_oauth = next( - container for container in agent_pod["containers"] - if container["name"] == "oauth2-proxy" - ) - assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in agent_oauth["args"] - assert { - "name": "oauth-templates", - "mountPath": "/etc/oauth2-proxy/templates", - "readOnly": True, - } in agent_oauth["volumeMounts"] - agent_template_volume = next( - volume for volume in agent_pod["volumes"] - if volume["name"] == "oauth-templates" - ) - assert agent_template_volume["configMap"]["name"] == ( - "hermes-chat-oauth-templates" - ) - - -def test_webui_recovers_auth_and_labels_session_scoped_controls(): - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() - - assert "res.status===401||res.status===403" in dockerfile - assert "window.location.assign('/oauth2/start?rd='" in dockerfile - assert "childrenExpanded?'▾ ':'▸ '" in dockerfile - assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile - assert "hermes-webui-telegram-project-patch.py" in dockerfile - telegram_project_patch = ( - ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py" - ).read_text() - assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch - assert "session_key.startswith('telegram-topic-')" in telegram_project_patch - - router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text() - assert "'atlas/auto/fast':'AUTO · Fast'" in router - assert "'atlas/manual/codex/sol':'Codex · SOL'" in router - assert "'atlas/manual/claude/opus':'Claude · Opus'" in router - assert "watchModelOptions('modelSelect')" in router - assert "watchModelOptions('settingsModel')" in router - - -def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - containers = statefulset["spec"]["template"]["spec"]["containers"] - webui = next(item for item in containers if item["name"] == "webui") - env = {item["name"]: item["value"] for item in webui["env"]} - - assert env["HERMES_STT_URL"] == ( - "http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions" - ) - assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == ( - "http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech" - ) - assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"] - - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["stt"] == { - "enabled": True, - "provider": "local_command", - "local": {"model": "small", "language": "auto"}, - } - - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() - assert "hermes-webui-atlas-patch.py" in dockerfile - assert "hermes-webui-atlas-voice.js" in dockerfile - voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text() - assert "/api/transcribe/capability" in voice_script - assert "/api/transcribe" in voice_script - assert "/api/tts" in voice_script - assert "speakResponse(generation)" in voice_script - assert "window._splitForTTS(text,280)" in voice_script - assert "pending=fetchSpeech(chunks[index+1])" in voice_script - assert "restartSoon(token,450)" in voice_script - assert "constraints.voiceIsolation=true" in voice_script - assert "highpass.frequency.value=140" in voice_script - assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script - assert "while(preRoll.length>3) preRoll.shift()" in voice_script - - stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() - assert "def _repetitive_token" in stt_server - assert "compression_ratio_threshold=2.0" in stt_server - assert "no_speech_threshold=0.5" in stt_server - - -def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch): - server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py" - spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace()) - monkeypatch.setitem( - sys.modules, - "torch", - SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)), - ) - monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace()) - spec.loader.exec_module(module) - - result = { - "segments": [ - { - "text": " ththththththththth Testing.", - "no_speech_prob": 0.12, - "avg_logprob": -0.2, - }, - { - "text": " background hum", - "no_speech_prob": 0.82, - "avg_logprob": -0.9, - }, - ] - } - - assert module._clean_transcript(result) == "Testing." - - -def test_voice_models_are_baked_and_runtime_has_no_public_egress(): - stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text() - tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text() - assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile - assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile - assert "--chmod=0444" in stt_dockerfile - assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile - assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile - assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile - assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile - assert tts_dockerfile.count("--chmod=0444") == 6 - assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile - assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile - tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text() - assert "download_voice" not in tts_server - assert "baked Piper voice is missing" in tts_server - assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server - - policies = _documents(HERMES / "networkpolicy.yaml") - voice_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-private-voice" - ) - assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"] - assert not any( - "ipBlock" in destination - for rule in voice_policy["spec"]["egress"] - for destination in rule.get("to", []) - ) - - -def test_voice_workloads_have_deliberate_xavier_placement(): - documents = _documents(HERMES / "voice-deployment.yaml") - deployments = { - item["metadata"]["name"]: item - for item in documents - if item["kind"] == "Deployment" - } - stt = deployments["hermes-stt"]["spec"]["template"]["spec"] - tts = deployments["hermes-tts"]["spec"]["template"]["spec"] - - assert "@sha256:" in stt["containers"][0]["image"] - assert "@sha256:" in tts["containers"][0]["image"] - assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} - assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} - assert stt["automountServiceAccountToken"] is False - assert tts["automountServiceAccountToken"] is False - assert stt["enableServiceLinks"] is False - assert tts["enableServiceLinks"] is False - assert stt["runtimeClassName"] == "nvidia" - assert stt["securityContext"]["supplementalGroups"] == [44] - stt_resources = stt["containers"][0]["resources"] - stt_env = { - item["name"]: item["value"] for item in stt["containers"][0]["env"] - } - assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility" - assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1 - assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1 - assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"] - tts_env = { - item["name"]: item["value"] for item in tts["containers"][0]["env"] - } - assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium" - assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2" - assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" - assert all("hostPath" not in volume for volume in stt["volumes"]) - assert all("hostPath" not in volume for volume in tts["volumes"]) - - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) - - -def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): - """Follow-up edits resolve the source server-side and keep tool JSON small.""" - provider_module = SimpleNamespace( - DEFAULT_ASPECT_RATIO="square", - ImageGenProvider=object, - error_response=lambda **value: value, - normalize_reference_images=lambda value: value, - resolve_aspect_ratio=lambda value: value, - save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", - success_response=lambda **value: value, - ) - monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) - monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) - spec = importlib.util.spec_from_file_location( - "hermes_image_plugin", - HERMES / "plugins" / "image-gen-broker" / "__init__.py", - ) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - image_dir = tmp_path / "cache" / "images" - image_dir.mkdir(parents=True) - older = image_dir / "atlas_flux-old.png" - newest = image_dir / "atlas_gpt-image-new.png" - older.write_bytes(b"older") - newest.write_bytes(b"newest") - older.touch() - time.sleep(0.001) - newest.touch() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - calls = [] - monkeypatch.setattr( - module, - "_handle_image_generate", - lambda args, route: calls.append((args, route)) or "ok", - ) - assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" - assert calls == [ - ( - { - "prompt": "make it a clown", - "image_url": str(newest.resolve()), - }, - "hosted", - ) - ] - - -def test_chat_reasoning_uses_switchyard_without_owner_credentials(): - """Family pods use AUTO/manual routes without mounting owner credentials.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/fast", - "model": "atlas/auto/fast", - } - assert config["providers"]["atlas-switchyard"] == { - "name": "Automatic Router", - "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", - "api_key": "atlas-switchyard", - "default_model": "atlas/auto/fast", - "transport": "chat_completions", - } - assert config["platforms"]["api_server"]["extra"]["model_routes"] == { - route: {"provider": "atlas-switchyard", "model": route} - for route in [ - "atlas/auto/fast", - "atlas/auto/balanced", - "atlas/auto/deep", - "atlas/auto/maximum", - "atlas/manual/codex/luna", - "atlas/manual/codex/terra", - "atlas/manual/codex/sol", - "atlas/manual/claude/haiku", - "atlas/manual/claude/fable", - "atlas/manual/claude/sonnet", - "atlas/manual/claude/opus", - "atlas/manual/local/qwen-14b", - ] - } - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "codex-broker") - assert broker["ports"] == [ - {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { - "PYTHONPATH": "/opt/hermes", - "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", - "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", - }.items() - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-codex-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - assert service["spec"]["ports"] == [ - { - "name": "http", - "port": 9003, - "targetPort": "codex-broker", - "protocol": "TCP", - } - ] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" - pod_spec = statefulset["spec"]["template"]["spec"] - patch_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-stream-recovery" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/conversation_loop.py", - "/patched/conversation_loop.py", - ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) - assert { - "name": "stream-recovery-patch", - "mountPath": "/opt/hermes/agent/conversation_loop.py", - "subPath": "conversation_loop.py", - } in hermes["volumeMounts"] - api_session_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-api-server-sessions" - ) - assert "patch_api_server_sessions.py" in api_session_init["args"][0] - assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] - assert { - "name": "api-server-patch", - "mountPath": "/opt/hermes/gateway/platforms/api_server.py", - "subPath": "api_server.py", - } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) - assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - - -def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): - """Only named Telegram conversations receive presentation metadata.""" - module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - state = tmp_path / "state.db" - responses = tmp_path / "response_store.db" - with sqlite3.connect(state) as connection: - connection.execute( - """CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, - display_name TEXT, origin_json TEXT, title TEXT - )""" - ) - connection.executemany( - "INSERT INTO sessions (id, source) VALUES (?, ?)", - (("telegram-session", "api_server"), ("other-session", "api_server")), - ) - with sqlite3.connect(responses) as connection: - connection.execute( - "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" - ) - connection.execute( - "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" - ) - connection.executemany( - "INSERT INTO responses VALUES (?, ?, 0)", - ( - ("telegram-response", json.dumps({"session_id": "telegram-session"})), - ("other-response", json.dumps({"session_id": "other-session"})), - ), - ) - connection.executemany( - "INSERT INTO conversations VALUES (?, ?)", - (("telegram", "telegram-response"), ("unrelated", "other-response")), - ) - - assert module.migrate(state, responses) == 1 - assert module.migrate(state, responses) == 0 - with sqlite3.connect(state) as connection: - telegram = connection.execute( - "SELECT session_key, chat_type, display_name, origin_json, title " - "FROM sessions WHERE id = 'telegram-session'" - ).fetchone() - other = connection.execute( - "SELECT session_key, title FROM sessions WHERE id = 'other-session'" - ).fetchone() - assert telegram[:3] == ("telegram", "private", "Telegram") - assert json.loads(telegram[3]) == { - "platform": "telegram", - "session_key": "telegram", - } - assert telegram[4] == "Telegram · General" - assert other == (None, None) - - -def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): - """The DOM transcript includes events without polluting agent history.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "web_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.LATEST_ROWS_BEFORE - + module.LATEST_SELECTION_BEFORE - + module.HELPER_MARKER - + " db = object()\n" - + module.MESSAGES_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "def _run_activity_messages(" in patched - assert 'root = get_hermes_home() / "run-activity"' in patched - assert "path.stat().st_size > 600_000" in patched - assert "entries[-1_000:]" in patched - assert "*_run_activity_messages(sid)" in patched - assert "messages.sort(" in patched - assert "limit: Optional[int] = None" in patched - assert "total_messages = len(messages)" in patched - assert "min(int(limit), 10_000)" in patched - assert '"total_messages": total_messages' in patched - assert "SELECT id, parent_session_id, started_at, ended_at" in patched - assert "newest still-open member of the lineage" in patched - assert "immediate objective parent" in patched - assert "mixes unrelated workstreams" in patched - assert "orphaned children" in patched - assert 'item[0].get("ended_at") is None' in patched - - -def test_web_session_lineage_returns_to_resumed_parent(): - """An ended reviewer must not strand the live view away from its parent.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - namespace: dict[str, object] = {} - exec( - "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, - namespace, - ) - - class FakeDB: - def get_session(self, session_id): - return next((row for row in rows if row["id"] == session_id), None) - - rows = [ - { - "id": "umbrella", - "parent_session_id": None, - "started_at": 0.0, - "ended_at": None, - }, - { - "id": "root", - "parent_session_id": "umbrella", - "started_at": 1.0, - "ended_at": None, - }, - { - "id": "review-1", - "parent_session_id": "root", - "started_at": 2.0, - "ended_at": 3.0, - }, - { - "id": "review-2", - "parent_session_id": "root", - "started_at": 4.0, - "ended_at": None, - }, - ] - select_active = namespace["select_active"] - - assert select_active("root", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - assert select_active("review-1", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - rows[2]["ended_at"] = 5.0 - rows[3]["ended_at"] = 5.0 - assert select_active("root", FakeDB(), rows) == ("root", ["root"]) - - rows[1]["ended_at"] = 9.0 - rows.append( - { - "id": "orphaned-review", - "parent_session_id": "root", - "started_at": 6.0, - "ended_at": None, - } - ) - rows.append( - { - "id": "accepted-review", - "parent_session_id": "root", - "started_at": 7.0, - "ended_at": 8.0, - } - ) - assert select_active("review-1", FakeDB(), rows) == ( - "accepted-review", - ["root", "accepted-review"], - ) - - -def test_api_activity_patch_coalesces_streaming_heartbeats( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """Token callbacks stay bounded while every tool transition is retained.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( - " def _make_run_event_callback(", 1 - )[0] - namespace: dict[str, object] = {} - exec( - "import hashlib, json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, - namespace, - ) - recorder = namespace["ActivityRecorder"]() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - recorder._record_run_activity("session", "_thinking") - recorder._record_run_activity("session", "reasoning.available") - recorder._record_run_activity("session", "subagent.thinking") - recorder._record_run_activity("session", "tool.started", tool_name="terminal") - recorder._record_run_activity("session", "tool.completed", tool_name="terminal") - - journals = list((tmp_path / "run-activity").glob("*.jsonl")) - assert len(journals) == 1 - entries = [json.loads(line) for line in journals[0].read_text().splitlines()] - assert [entry["activity_event"] for entry in entries] == [ - "_thinking", - "tool.started", - "tool.completed", - ] - - -def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): - """Known standalone API workers move under Cassandra without data loss.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", - (module.LEGACY_CASSANDRA_PARENT,), - ) - worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", - (worker_id,), - ) - connection.executemany( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", - ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ) - - assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - row = connection.execute( - "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", - (worker_id,), - ).fetchone() - assert row == ( - module.LEGACY_CASSANDRA_PARENT, - module.LEGACY_CASSANDRA_WORKERS[worker_id], - "keep-me", - ) - with sqlite3.connect(database) as connection: - orphans = connection.execute( - "SELECT id, archived, title, transcript FROM sessions " - "WHERE id IN ({}) ORDER BY id".format( - ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) - ), - tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ).fetchall() - assert orphans == sorted( - ( - orphan_id, - 1, - module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], - "keep-smoke", - ) - for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS - ) - - -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): - """Only the stable Ariadne contract moves below the triage parent.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" - ) - connection.executemany( - "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", - (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), - ) - connection.executemany( - "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", - ( - ( - "triage-run", - module.TRIAGE_MESSAGE_PREFIXES[0] - + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", - ), - ( - "jenkins-run", - module.TRIAGE_MESSAGE_PREFIXES[1] - + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", - ), - ("interactive-run", "Please explain this alert to me."), - ), - ) - - assert module.migrate(database, group_triage=True) == 2 - assert module.migrate(database, group_triage=True) == 0 - with sqlite3.connect(database) as connection: - parent = connection.execute( - "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) - ).fetchone() - triage = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" - ).fetchone() - interactive = connection.execute( - "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" - ).fetchone() - jenkins = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" - ).fetchone() - assert parent == (module.TRIAGE_PARENT_TITLE,) - assert triage == ( - module.TRIAGE_PARENT, - "Sonar · bstein_home · python:S2208 · iage-run", - ) - assert interactive == (None,) - assert jenkins == ( - module.TRIAGE_PARENT, - "Sonar · soteria/291 · kins-run", - ) - - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) - - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) - - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" - - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_sandbox.py b/testing/tests/test_hermes_chat_sandbox.py new file mode 100644 index 00000000..fb6856d1 --- /dev/null +++ b/testing/tests/test_hermes_chat_sandbox.py @@ -0,0 +1,24 @@ +"""Hermes chat sandbox contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + Path, + ROOT, + importlib, +) + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_sessions.py b/testing/tests/test_hermes_chat_sessions.py new file mode 100644 index 00000000..c4e2d6f2 --- /dev/null +++ b/testing/tests/test_hermes_chat_sessions.py @@ -0,0 +1,392 @@ +"""Hermes chat sessions contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + Path, + importlib, + json, + pytest, + sqlite3, +) + + +def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): + """Only named Telegram conversations receive presentation metadata.""" + module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions (id, source) VALUES (?, ?)", + (("telegram-session", "api_server"), ("other-session", "api_server")), + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("telegram-response", json.dumps({"session_id": "telegram-session"})), + ("other-response", json.dumps({"session_id": "other-session"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "telegram-response"), ("unrelated", "other-response")), + ) + + assert module.migrate(state, responses) == 1 + assert module.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + telegram = connection.execute( + "SELECT session_key, chat_type, display_name, origin_json, title " + "FROM sessions WHERE id = 'telegram-session'" + ).fetchone() + other = connection.execute( + "SELECT session_key, title FROM sessions WHERE id = 'other-session'" + ).fetchone() + assert telegram[:3] == ("telegram", "private", "Telegram") + assert json.loads(telegram[3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert telegram[4] == "Telegram · General" + assert other == (None, None) + +def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): + """The DOM transcript includes events without polluting agent history.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "web_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.LATEST_ROWS_BEFORE + + module.LATEST_SELECTION_BEFORE + + module.HELPER_MARKER + + " db = object()\n" + + module.MESSAGES_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "def _run_activity_messages(" in patched + assert 'root = get_hermes_home() / "run-activity"' in patched + assert "path.stat().st_size > 600_000" in patched + assert "entries[-1_000:]" in patched + assert "*_run_activity_messages(sid)" in patched + assert "messages.sort(" in patched + assert "limit: Optional[int] = None" in patched + assert "total_messages = len(messages)" in patched + assert "min(int(limit), 10_000)" in patched + assert '"total_messages": total_messages' in patched + assert "SELECT id, parent_session_id, started_at, ended_at" in patched + assert "newest still-open member of the lineage" in patched + assert "immediate objective parent" in patched + assert "mixes unrelated workstreams" in patched + assert "orphaned children" in patched + assert 'item[0].get("ended_at") is None' in patched + +def test_web_session_lineage_returns_to_resumed_parent(): + """An ended reviewer must not strand the live view away from its parent.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + namespace: dict[str, object] = {} + exec( + "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, + namespace, + ) + + class FakeDB: + def get_session(self, session_id): + return next((row for row in rows if row["id"] == session_id), None) + + rows = [ + { + "id": "umbrella", + "parent_session_id": None, + "started_at": 0.0, + "ended_at": None, + }, + { + "id": "root", + "parent_session_id": "umbrella", + "started_at": 1.0, + "ended_at": None, + }, + { + "id": "review-1", + "parent_session_id": "root", + "started_at": 2.0, + "ended_at": 3.0, + }, + { + "id": "review-2", + "parent_session_id": "root", + "started_at": 4.0, + "ended_at": None, + }, + ] + select_active = namespace["select_active"] + + assert select_active("root", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + assert select_active("review-1", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + rows[2]["ended_at"] = 5.0 + rows[3]["ended_at"] = 5.0 + assert select_active("root", FakeDB(), rows) == ("root", ["root"]) + + rows[1]["ended_at"] = 9.0 + rows.append( + { + "id": "orphaned-review", + "parent_session_id": "root", + "started_at": 6.0, + "ended_at": None, + } + ) + rows.append( + { + "id": "accepted-review", + "parent_session_id": "root", + "started_at": 7.0, + "ended_at": 8.0, + } + ) + assert select_active("review-1", FakeDB(), rows) == ( + "accepted-review", + ["root", "accepted-review"], + ) + +def test_api_activity_patch_coalesces_streaming_heartbeats( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Token callbacks stay bounded while every tool transition is retained.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( + " def _make_run_event_callback(", 1 + )[0] + namespace: dict[str, object] = {} + exec( + "import hashlib, json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def redact_sensitive_text(value): return value\n" + "class ActivityRecorder:\n" + + activity_body, + namespace, + ) + recorder = namespace["ActivityRecorder"]() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + recorder._record_run_activity("session", "_thinking") + recorder._record_run_activity("session", "reasoning.available") + recorder._record_run_activity("session", "subagent.thinking") + recorder._record_run_activity("session", "tool.started", tool_name="terminal") + recorder._record_run_activity("session", "tool.completed", tool_name="terminal") + + journals = list((tmp_path / "run-activity").glob("*.jsonl")) + assert len(journals) == 1 + entries = [json.loads(line) for line in journals[0].read_text().splitlines()] + assert [entry["activity_event"] for entry in entries] == [ + "_thinking", + "tool.started", + "tool.completed", + ] + +def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): + """Known standalone API workers move under Cassandra without data loss.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", + (module.LEGACY_CASSANDRA_PARENT,), + ) + worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", + (worker_id,), + ) + connection.executemany( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", + ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ) + + assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + row = connection.execute( + "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", + (worker_id,), + ).fetchone() + assert row == ( + module.LEGACY_CASSANDRA_PARENT, + module.LEGACY_CASSANDRA_WORKERS[worker_id], + "keep-me", + ) + with sqlite3.connect(database) as connection: + orphans = connection.execute( + "SELECT id, archived, title, transcript FROM sessions " + "WHERE id IN ({}) ORDER BY id".format( + ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) + ), + tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ).fetchall() + assert orphans == sorted( + ( + orphan_id, + 1, + module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], + "keep-smoke", + ) + for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS + ) + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): + """Only the stable Ariadne contract moves below the triage parent.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" + ) + connection.executemany( + "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", + (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), + ) + connection.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", + ( + ( + "triage-run", + module.TRIAGE_MESSAGE_PREFIXES[0] + + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", + ), + ( + "jenkins-run", + module.TRIAGE_MESSAGE_PREFIXES[1] + + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", + ), + ("interactive-run", "Please explain this alert to me."), + ), + ) + + assert module.migrate(database, group_triage=True) == 2 + assert module.migrate(database, group_triage=True) == 0 + with sqlite3.connect(database) as connection: + parent = connection.execute( + "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) + ).fetchone() + triage = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" + ).fetchone() + interactive = connection.execute( + "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" + ).fetchone() + jenkins = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" + ).fetchone() + assert parent == (module.TRIAGE_PARENT_TITLE,) + assert triage == ( + module.TRIAGE_PARENT, + "Sonar · bstein_home · python:S2208 · iage-run", + ) + assert interactive == (None,) + assert jenkins == ( + module.TRIAGE_PARENT, + "Sonar · soteria/291 · kins-run", + ) + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py new file mode 100644 index 00000000..7f09512c --- /dev/null +++ b/testing/tests/test_hermes_chat_support.py @@ -0,0 +1,73 @@ +"""Contracts for isolated high-quality Hermes chat capabilities.""" + +from __future__ import annotations + +import base64 + +import importlib.util + +import json + +import sqlite3 + +import sys + +import time + +import tomllib + +from pathlib import Path + +from types import ModuleType, SimpleNamespace + +import pytest + +import yaml + +ROOT = Path(__file__).parents[2] + +HERMES = ROOT / "services" / "hermes" + +VAULT = ROOT / "services" / "vault" + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +__all__ = ( + "HERMES", + "ModuleType", + "Path", + "ROOT", + "SimpleNamespace", + "VAULT", + "_documents", + "_load_broker_module", + "base64", + "importlib", + "json", + "pytest", + "sqlite3", + "sys", + "time", + "tomllib", + "yaml", +) diff --git a/testing/tests/test_hermes_chat_switchyard.py b/testing/tests/test_hermes_chat_switchyard.py new file mode 100644 index 00000000..6f8e3de7 --- /dev/null +++ b/testing/tests/test_hermes_chat_switchyard.py @@ -0,0 +1,483 @@ +"""Hermes chat switchyard contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + Path, + SimpleNamespace, + _documents, + _load_broker_module, + base64, + importlib, + json, + pytest, + sys, + time, + tomllib, +) + + +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module( + "hermes_codex_broker", "codex_broker.py", monkeypatch + ) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": response_item} + )["input"] == [response_item] + response_items = [response_item] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": response_items} + )["input"] is response_items + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"][ + "output" + ] == [completed_item] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ + "output" + ] == [] + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = base64.urlsafe_b64encode( + json.dumps({"exp": time.time() + 3600}).encode() + ).decode().rstrip("=") + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" diff --git a/testing/tests/test_hermes_chat_voice.py b/testing/tests/test_hermes_chat_voice.py new file mode 100644 index 00000000..180a17e4 --- /dev/null +++ b/testing/tests/test_hermes_chat_voice.py @@ -0,0 +1,265 @@ +"""Hermes chat voice contracts.""" + +from __future__ import annotations + +from test_hermes_chat_support import ( + HERMES, + ROOT, + SimpleNamespace, + VAULT, + _documents, + importlib, + sys, + yaml, +) + + +def test_webui_recovers_auth_and_labels_session_scoped_controls(): + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() + + assert "res.status===401||res.status===403" in dockerfile + assert "window.location.assign('/oauth2/start?rd='" in dockerfile + assert "childrenExpanded?'▾ ':'▸ '" in dockerfile + assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile + assert "hermes-webui-telegram-project-patch.py" in dockerfile + telegram_project_patch = ( + ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py" + ).read_text() + assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch + assert "session_key.startswith('telegram-topic-')" in telegram_project_patch + + router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text() + assert "'atlas/auto/fast':'AUTO · Fast'" in router + assert "'atlas/manual/codex/sol':'Codex · SOL'" in router + assert "'atlas/manual/claude/opus':'Claude · Opus'" in router + assert "watchModelOptions('modelSelect')" in router + assert "watchModelOptions('settingsModel')" in router + +def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + containers = statefulset["spec"]["template"]["spec"]["containers"] + webui = next(item for item in containers if item["name"] == "webui") + env = {item["name"]: item["value"] for item in webui["env"]} + + assert env["HERMES_STT_URL"] == ( + "http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions" + ) + assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == ( + "http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech" + ) + assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"] + + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["stt"] == { + "enabled": True, + "provider": "local_command", + "local": {"model": "small", "language": "auto"}, + } + + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() + assert "hermes-webui-atlas-patch.py" in dockerfile + assert "hermes-webui-atlas-voice.js" in dockerfile + voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text() + assert "/api/transcribe/capability" in voice_script + assert "/api/transcribe" in voice_script + assert "/api/tts" in voice_script + assert "speakResponse(generation)" in voice_script + assert "window._splitForTTS(text,280)" in voice_script + assert "pending=fetchSpeech(chunks[index+1])" in voice_script + assert "restartSoon(token,450)" in voice_script + assert "constraints.voiceIsolation=true" in voice_script + assert "highpass.frequency.value=140" in voice_script + assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script + assert "while(preRoll.length>3) preRoll.shift()" in voice_script + + stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() + assert "def _repetitive_token" in stt_server + assert "compression_ratio_threshold=2.0" in stt_server + assert "no_speech_threshold=0.5" in stt_server + +def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch): + server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py" + spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace()) + monkeypatch.setitem( + sys.modules, + "torch", + SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)), + ) + monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace()) + spec.loader.exec_module(module) + + result = { + "segments": [ + { + "text": " ththththththththth Testing.", + "no_speech_prob": 0.12, + "avg_logprob": -0.2, + }, + { + "text": " background hum", + "no_speech_prob": 0.82, + "avg_logprob": -0.9, + }, + ] + } + + assert module._clean_transcript(result) == "Testing." + +def test_voice_models_are_baked_and_runtime_has_no_public_egress(): + stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text() + tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text() + assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile + assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile + assert "--chmod=0444" in stt_dockerfile + assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile + assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile + assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile + assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile + assert tts_dockerfile.count("--chmod=0444") == 6 + assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile + assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile + tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text() + assert "download_voice" not in tts_server + assert "baked Piper voice is missing" in tts_server + assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server + + policies = _documents(HERMES / "networkpolicy.yaml") + voice_policy = next( + item for item in policies if item["metadata"]["name"] == "hermes-private-voice" + ) + assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"] + assert not any( + "ipBlock" in destination + for rule in voice_policy["spec"]["egress"] + for destination in rule.get("to", []) + ) + +def test_voice_workloads_have_deliberate_xavier_placement(): + documents = _documents(HERMES / "voice-deployment.yaml") + deployments = { + item["metadata"]["name"]: item + for item in documents + if item["kind"] == "Deployment" + } + stt = deployments["hermes-stt"]["spec"]["template"]["spec"] + tts = deployments["hermes-tts"]["spec"]["template"]["spec"] + + assert "@sha256:" in stt["containers"][0]["image"] + assert "@sha256:" in tts["containers"][0]["image"] + assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} + assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} + assert stt["automountServiceAccountToken"] is False + assert tts["automountServiceAccountToken"] is False + assert stt["enableServiceLinks"] is False + assert tts["enableServiceLinks"] is False + assert stt["runtimeClassName"] == "nvidia" + assert stt["securityContext"]["supplementalGroups"] == [44] + stt_resources = stt["containers"][0]["resources"] + stt_env = { + item["name"]: item["value"] for item in stt["containers"][0]["env"] + } + assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility" + assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1 + assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1 + assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"] + tts_env = { + item["name"]: item["value"] for item in tts["containers"][0]["env"] + } + assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium" + assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2" + assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" + assert all("hostPath" not in volume for volume in stt["volumes"]) + assert all("hostPath" not in volume for volume in tts["volumes"]) + +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + assert ( + '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' + 'hermes/developer-keycloak hermes/developer-gitea ' + 'hermes/developer-harbor hermes/developer-jenkins ' + 'hermes/developer-ssh"' + in vault_policy + ) + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) diff --git a/testing/tests/test_hermes_cli_dispatch.py b/testing/tests/test_hermes_cli_dispatch.py new file mode 100644 index 00000000..4138ab63 --- /dev/null +++ b/testing/tests/test_hermes_cli_dispatch.py @@ -0,0 +1,420 @@ +"""Board scanning, claiming, orphan recovery, and replay guards.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + json, + lanes, + nullcontext, + pytest, + sys, +) + + +def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): + task = SimpleNamespace(id="t_auto", assignee=None, status="ready") + assigned = [] + + class Connection: + def close(self): + return None + + def assign_task(_conn, task_id, profile): + assigned.append((task_id, profile)) + task.assignee = profile + return True + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + assign_task=assign_task, + get_task=lambda _conn, _task_id: task, + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] + assert assigned == [("t_auto", "cli-auto")] + + +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err + + +def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + def recompute_ready(connection): + if connection.board == "cassandra": + raise lanes.sqlite3.OperationalError("disk I/O error") + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(board), + recompute_ready=recompute_ready, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + error = capsys.readouterr().err + assert "temporarily skipping Kanban board 'cassandra'" in error + assert "storage OperationalError: disk I/O error" in error + + +def test_board_call_retries_storage_faults_on_fresh_connections(): + connections = [] + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + attempts = [] + + def operation(_connection): + attempts.append(1) + if len(attempts) < 3: + raise lanes.sqlite3.OperationalError("disk I/O error") + return "healthy" + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + ) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" + assert len(connections) == 3 + assert all(connection.closed for connection in connections) + + +def test_accepted_result_survives_failed_finalization_and_replays_exact_run( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_terminal", + status="running", + result=None, + current_run_id=41, + assignee="cli-auto", + max_runtime_seconds=60, + ) + completions = [] + comments = [] + permit_completion = {"value": False} + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + completions.append(kwargs) + if not permit_completion["value"]: + return False + task.status = "done" + task.result = kwargs["result"] + task.completed_run_id = kwargs["expected_run_id"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish and verify the objective.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **_kwargs: pytest.fail( + "an accepted journaled result must not be converted into a block" + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + structured = { + "status": "completed", + "summary": "Verified exact terminal result.", + "changed_files": [], + "tests_run": ["pytest: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + } + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False), + ) + + lanes.execute_claim("cassandra", "t_terminal") + + candidates = list((state_root / "cassandra").glob("*.candidate-*.json")) + terminals = list((state_root / "cassandra").glob("*.terminal.pending.json")) + assert len(candidates) == 1 + assert json.loads(candidates[0].read_text())["structured"] == structured + assert len(terminals) == 1 + assert json.loads(terminals[0].read_text())["kanban_state"] == "pending" + assert task.status == "running" + assert any("durably journaled" in body for body in comments) + + permit_completion["value"] = True + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert len(completions) == 2 + assert not terminals[0].exists() + committed = list( + (state_root / "cassandra").glob("*.terminal.committed.json") + ) + assert len(committed) == 1 + terminal = json.loads(committed[0].read_text()) + assert terminal["kanban_state"] == "committed" + assert completions[-1]["result"] == terminal["result"] + + +def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + state_file = lanes.state_path("cassandra", "t_terminal") + terminal_path, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_terminal", + run_id=8, + structured={ + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + summary="done", + metadata={}, + ) + assert terminal_path.exists() + task = SimpleNamespace( + id="t_terminal", + status="running", + assignee="cli-auto", + current_run_id=8, + ) + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + list_tasks=lambda _conn: [task], + reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + + lanes.recover_orphans() + + assert reclaimed == [] + + +def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + tasks = [ + SimpleNamespace( + id="t_guarded", + status="running", + assignee="cli-auto", + current_run_id=91, + ), + SimpleNamespace( + id="t_no_run", + status="running", + assignee="cli-auto", + current_run_id=None, + ), + ] + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + list_tasks=lambda _conn: tasks, + reclaim_task=lambda _conn, task_id, **kwargs: ( + reclaimed.append((task_id, kwargs)) or True + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + + lanes.recover_orphans() + + assert reclaimed == [ + ( + "t_guarded", + { + "reason": "direct CLI lane restarted; provider session will resume", + "expected_run_id": 91, + }, + ) + ] + + +def test_prepared_evidence_without_pending_still_pins_the_exact_run( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + pending, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_prepared_pin"), + board="cassandra", + task_id="t_prepared_pin", + run_id=14, + structured=_completed_result("accepted and prepared"), + summary="accepted and prepared", + metadata={}, + ) + identity = lanes._terminal_identity(pending) + assert identity is not None + lanes._persist_prepared_evidence(identity, record) + pending.unlink() + + assert lanes._has_pending_finalization("cassandra", "t_prepared_pin", 14) is True + + +def test_terminal_replay_never_crosses_into_a_replacement_run( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + state_file = lanes.state_path("cassandra", "t_terminal") + terminal_path, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_terminal", + run_id=8, + structured={ + "status": "completed", + "summary": "old run", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + summary="old run", + metadata={}, + ) + replacement = SimpleNamespace( + id="t_terminal", + status="running", + result=None, + current_run_id=9, + ) + completions = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: replacement, + complete_task=lambda *_args, **_kwargs: completions.append(True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + assert completions == [] + assert not terminal_path.exists() + conflicts = list((state_root / "cassandra").glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + conflict = json.loads(conflicts[0].read_text(encoding="utf-8")) + assert conflict["result"] == _record["result"] + assert conflict["kanban_state"] == "conflict" diff --git a/testing/tests/test_hermes_cli_durability.py b/testing/tests/test_hermes_cli_durability.py new file mode 100644 index 00000000..ceca8473 --- /dev/null +++ b/testing/tests/test_hermes_cli_durability.py @@ -0,0 +1,361 @@ +"""Prepared evidence and filesystem durability boundaries.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + errno, + json, + lanes, + nullcontext, + os, + pytest, + stat, + sys, + threading, +) + + +def test_unknown_db_completion_outcome_replays_from_prepared_evidence( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_unknown_db_outcome"), + board="cassandra", + task_id="t_unknown_db_outcome", + run_id=35, + structured=_completed_result("DB committed before transport error"), + summary="DB committed before transport error", + metadata={}, + ) + task = SimpleNamespace( + id="t_unknown_db_outcome", + status="running", + result=None, + current_run_id=35, + completed_run_id=None, + assignee="cli-auto", + ) + fail_once = {"value": True} + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.completed_run_id = kwargs["expected_run_id"] + task.current_run_id = None + if fail_once["value"]: + fail_once["value"] = False + raise RuntimeError("transport failed after DB commit") + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + + with pytest.raises(RuntimeError, match="after DB commit"): + lanes._finalize_terminal_record(fake_db, path, record) + + assert task.status == "done" + assert path.exists() + assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1 + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + assert lanes.recover_pending_finalizations() == 1 + assert not path.exists() + assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] + committed = list(path.parent.glob("*.terminal.committed.json")) + assert len(committed) == 1 + assert json.loads(committed[0].read_text())["result"] == record["result"] + + +def test_pending_retirement_fsync_failure_keeps_db_winner_committed( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_retire_fsync"), + board="cassandra", + task_id="t_retire_fsync", + run_id=33, + structured=_completed_result("retirement fsync winner"), + summary="retirement fsync winner", + metadata={}, + ) + task = SimpleNamespace( + id="t_retire_fsync", + status="running", + result=None, + current_run_id=33, + assignee="cli-auto", + ) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + real_fsync = lanes.os.fsync + failed = {"value": False} + + def fail_pending_retirement_fsync(descriptor): + if ( + stat.S_ISDIR(os.fstat(descriptor).st_mode) + and not path.exists() + and list(path.parent.glob("*.terminal.committed.json")) + and not failed["value"] + ): + failed["value"] = True + raise OSError(errno.ENOSPC, "pending retirement fsync failed") + real_fsync(descriptor) + + monkeypatch.setattr(lanes.os, "fsync", fail_pending_retirement_fsync) + + assert lanes._finalize_terminal_record(fake_db, path, record) == "committed" + assert failed["value"] is True + assert task.status == "done" + assert task.result == record["result"] + assert not path.exists() + assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1 + assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] + + +def test_committed_first_writer_is_never_overwritten_by_a_db_winner( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_committed_collision") + pending, winner = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_committed_collision", + run_id=34, + structured=_completed_result("DB winner"), + summary="DB winner", + metadata={"writer": "db"}, + ) + committed_path = lanes._terminal_path(state_file, 34, "committed") + first_writer = { + "board": "cassandra", + "task_id": "t_committed_collision", + "expected_run_id": 34, + "result": json.dumps(_completed_result("evidence first writer"), sort_keys=True), + "summary": "evidence first writer", + "metadata": {"writer": "evidence"}, + "kanban_state": "committed", + "recorded_at": lanes.utc_now(), + } + lanes.atomic_json(committed_path, first_writer) + task = SimpleNamespace( + id="t_committed_collision", + status="running", + result=None, + current_run_id=34, + assignee="cli-auto", + ) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + + assert lanes._finalize_terminal_record(fake_db, pending, winner) == "committed" + assert task.result == winner["result"] + assert json.loads(committed_path.read_text())["result"] == first_writer["result"] + conflicts = list(pending.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + assert json.loads(conflicts[0].read_text())["result"] == winner["result"] + assert not pending.exists() + + +def test_concurrent_duplicate_terminal_finalizers_are_idempotent( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_duplicate"), + board="cassandra", + task_id="t_duplicate", + run_id=29, + structured=_completed_result("same accepted result"), + summary="same accepted result", + metadata={}, + ) + task_state = { + "status": "running", + "result": None, + "current_run_id": 29, + "completed_run_id": None, + } + state_lock = threading.Lock() + readers = threading.Barrier(2) + completions = [] + + class Connection: + def close(self): + return None + + def get_task(_conn, _task_id): + with state_lock: + snapshot = SimpleNamespace( + id="t_duplicate", + assignee="cli-auto", + **task_state, + ) + if snapshot.status == "running": + readers.wait(timeout=5) + return snapshot + + def complete_task(_conn, _task_id, **kwargs): + with state_lock: + if task_state["status"] != "running": + return False + task_state.update( + status="done", + result=kwargs["result"], + current_run_id=None, + completed_run_id=kwargs["expected_run_id"], + ) + completions.append(kwargs["result"]) + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=get_task, + complete_task=complete_task, + ) + outcomes = [] + errors = [] + + def finalize(): + try: + outcomes.append(lanes._finalize_terminal_record(fake_db, path, record)) + except Exception as error: # pragma: no cover - assertion reports detail + errors.append(error) + + workers = [threading.Thread(target=finalize) for _index in range(2)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=10) + + assert all(not worker.is_alive() for worker in workers) + assert errors == [] + assert outcomes == ["committed", "committed"] + assert len(completions) == 1 + assert not path.exists() + assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1 + assert list(path.parent.glob("*.terminal.conflict-*.json")) == [] + assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] + + +def test_recovery_uses_first_durable_prepared_result_as_db_writer( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_prepared_order") + pending, first = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_prepared_order", + run_id=30, + structured=_completed_result("first durable result"), + summary="first durable result", + metadata={"writer": "first"}, + ) + identity = lanes._terminal_identity(pending) + assert identity is not None + first_prepared = lanes._persist_prepared_evidence(identity, first) + os.utime(first_prepared, ns=(100, 100)) + second_structured = _completed_result("second durable result") + second = { + "board": "cassandra", + "task_id": "t_prepared_order", + "expected_run_id": 30, + "result": json.dumps(second_structured, sort_keys=True), + "summary": "second durable result", + "metadata": {"writer": "second"}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + second_prepared = lanes._persist_prepared_evidence(identity, second) + os.utime(second_prepared, ns=(200, 200)) + lanes.atomic_json(pending, second) + task = SimpleNamespace( + id="t_prepared_order", + status="running", + result=None, + current_run_id=30, + assignee="cli-auto", + ) + completions = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + if task.status != "running": + return False + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + completions.append(kwargs["result"]) + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert completions == [first["result"]] + assert task.result == first["result"] + assert not pending.exists() + assert list(pending.parent.glob("*.terminal.prepared-*.json")) == [] + committed = list(pending.parent.glob("*.terminal.committed.json")) + conflicts = list(pending.parent.glob("*.terminal.conflict-*.json")) + assert len(committed) == 1 + assert len(conflicts) == 1 + assert json.loads(committed[0].read_text())["result"] == first["result"] + assert json.loads(conflicts[0].read_text())["result"] == second["result"] diff --git a/testing/tests/test_hermes_cli_fallback.py b/testing/tests/test_hermes_cli_fallback.py new file mode 100644 index 00000000..e2a8141e --- /dev/null +++ b/testing/tests/test_hermes_cli_fallback.py @@ -0,0 +1,381 @@ +"""Provider fallback, workspace, and command contracts.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + json, + lanes, + nullcontext, + pytest, + sys, +) + + +def test_goal_card_continues_after_local_judge_rejects_progress( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_goal", + status="running", + result=None, + current_run_id=12, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + calls = [] + comments = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () + ) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "Focused tests passed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "Full tests passed; commit pushed and remote HEAD verified.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter( + [ + (False, "commit, push, and remote verification are missing"), + (True, "all explicit acceptance criteria have evidence"), + ] + ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + judge_goal_completion, + ) + + lanes.execute_claim("cassandra", "t_goal") + + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) + assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] + candidates = sorted( + (lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json") + ) + assert len(candidates) == 2 + assert json.loads(candidates[0].read_text())["structured"]["summary"] == ( + "Focused tests passed." + ) + assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith( + "Full tests passed" + ) + assert reports == [] + + +def test_capacity_fallback_preserves_first_claude_structured_response( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_fallback", + status="running", + result=None, + current_run_id=14, + assignee="cli-auto", + max_runtime_seconds=60, + ) + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Complete with a fallback if needed.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda *_args: None, + complete_task=lambda _conn, _task_id, **kwargs: ( + setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True + ), + block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + claude = lanes.Route( + "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, () + ) + codex = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda _prompt, assignee, **_kwargs: codex + if assignee == "cli-codex-high" + else claude, + ) + reports = [ + lanes.ProcessResult( + 1, + "subscription capacity exhausted", + { + **_completed_result("Claude preserved evidence"), + "status": "incomplete", + "blockers": ["subscription capacity exhausted"], + }, + True, + ), + lanes.ProcessResult(0, "", _completed_result("Codex completed"), False), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + + lanes.execute_claim("cassandra", "t_fallback") + + candidates = sorted( + (lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json") + ) + assert len(candidates) == 2 + first, second = [json.loads(path.read_text()) for path in candidates] + assert (first["provider"], first["structured"]["summary"]) == ( + "claude", + "Claude preserved evidence", + ) + assert (second["provider"], second["structured"]["summary"]) == ( + "codex", + "Codex completed", + ) + assert task.status == "done" + + +def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): + task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") + calls = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( + ValueError("not a Git repository") + ), + block_task=lambda *_args, **kwargs: calls.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") + + lanes.execute_claim("cassandra", "t_bad_worktree") + + assert calls[0]["kind"] == "capability" + assert calls[0]["expected_run_id"] == 7 + assert "not a Git repository" in calls[0]["reason"] + + +def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): + workspace = tmp_path / "workspace" + workspace.mkdir() + inside = workspace / "report.json" + outside = tmp_path / "auth.json" + inside.write_text("{}\n", encoding="utf-8") + outside.write_text("secret\n", encoding="utf-8") + + assert lanes.workspace_artifacts( + workspace, + ["report.json", str(outside), "missing.json"], + ) == [str(inside)] + + +def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): + task = SimpleNamespace( + id="t_resume", + status="running", + result=None, + current_run_id=9, + assignee="cli-auto", + max_runtime_seconds=60, + ) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_resume") + state_file.parent.mkdir(parents=True) + state_file.write_text( + json.dumps({"current_route": {"provider": "claude"}}), + encoding="utf-8", + ) + (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") + prompts = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume objective", + add_comment=lambda *_args: None, + complete_task=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: f"HANDOFF:{output}", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda _route, prompt, *_args, **_kwargs: ( + prompts.append(prompt) + or lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ) + ), + ) + + lanes.execute_claim("cassandra", "t_resume") + + assert "HANDOFF:prior provider evidence" in prompts[0] + + +def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): + route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) + command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") + assert "--dangerously-bypass-approvals-and-sandbox" in command + assert "--json" in command + assert "--output-schema" in command + assert 'model_reasoning_effort="xhigh"' in command + + claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} + claude = lanes._claude_command( + lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), + "Review.", + claude_state, + False, + ) + assert "--dangerously-skip-permissions" in claude + assert "--output-format" in claude and "stream-json" in claude + assert "--json-schema" in claude + assert "--disallowedTools" in claude + assert "Bash(kubectl apply *)" not in claude + assert "Bash(flux reconcile *)" not in claude + assert "max" not in claude + + +def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): + prompt = lanes.build_prompt("Review the change.", tmp_path) + + assert "put defects and risks in findings" in prompt + assert "blockers array must be empty whenever status is completed" in prompt + assert "findings" in lanes.RESULT_SCHEMA["properties"] + assert set(lanes.RESULT_SCHEMA["required"]) == set( + lanes.RESULT_SCHEMA["properties"] + ) + assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] diff --git a/testing/tests/test_hermes_cli_finalization.py b/testing/tests/test_hermes_cli_finalization.py new file mode 100644 index 00000000..586240ad --- /dev/null +++ b/testing/tests/test_hermes_cli_finalization.py @@ -0,0 +1,411 @@ +"""Terminal-result validation and exact-run finalization.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + errno, + json, + lanes, + nullcontext, + pytest, + sys, +) + + +def test_terminal_recovery_can_complete_the_exact_latest_ended_run( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_ended") + pending, _record = lanes._write_terminal_record( + state_file, + board="cassandra", + task_id="t_ended", + run_id=17, + structured=_completed_result("accepted before legacy block"), + summary="accepted before legacy block", + metadata={}, + ) + task = SimpleNamespace( + id="t_ended", + status="blocked", + result=None, + current_run_id=None, + assignee="cli-auto", + ) + guards = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + guards.append(kwargs) + task.status = "done" + task.result = kwargs["result"] + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert guards[0]["replay_ended_run_id"] == 17 + assert "expected_run_id" not in guards[0] + assert task.status == "done" + assert not pending.exists() + + +def test_terminal_recovery_completes_an_unchanged_triage_exact_run( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + pending, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_triage_ended"), + board="cassandra", + task_id="t_triage_ended", + run_id=18, + structured=_completed_result("accepted before loop-breaker triage"), + summary="accepted before loop-breaker triage", + metadata={}, + ) + task = SimpleNamespace( + id="t_triage_ended", + status="triage", + result=None, + current_run_id=None, + completed_run_id=None, + assignee="cli-auto", + ) + guards = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + guards.append(kwargs) + assert kwargs == { + "result": record["result"], + "summary": record["summary"], + "metadata": {}, + "replay_ended_run_id": 18, + } + task.status = "done" + task.result = kwargs["result"] + task.completed_run_id = 18 + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert len(guards) == 1 + assert task.status == "done" + assert task.completed_run_id == 18 + assert not pending.exists() + assert lanes.recover_pending_finalizations() == 0 + assert len(guards) == 1 + + +def test_complete_exception_after_journal_is_replayable_and_never_blocks( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_crash", + status="running", + result=None, + current_run_id=51, + assignee="cli-auto", + max_runtime_seconds=60, + ) + completion_raises = {"value": True} + blocks = [] + comments = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + if completion_raises["value"]: + raise RuntimeError("crash between journal and commit") + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish safely.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "", _completed_result("crash-safe result"), False + ), + ) + + lanes.execute_claim("cassandra", "t_crash") + + pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) + assert len(pending) == 1 + assert blocks == [] + assert any("RuntimeError" in body for body in comments) + + completion_raises["value"] = False + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert not pending[0].exists() + + +def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + task = SimpleNamespace( + id="t_enospc", + status="running", + result=None, + current_run_id=52, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + comments = [] + completions = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + completions.append(kwargs) + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish without losing the result.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=complete_task, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "", _completed_result("persisted before ENOSPC"), False + ), + ) + real_replace = lanes.os.replace + real_fsync_directory = lanes._fsync_directory + terminal_replaced = {"value": False} + fail_once = {"value": True} + + def replace_then_mark(source, destination): + real_replace(source, destination) + if str(destination).endswith(".terminal.pending.json"): + terminal_replaced["value"] = True + + def fail_after_terminal_replace(directory): + if terminal_replaced["value"] and fail_once["value"]: + fail_once["value"] = False + raise OSError(errno.ENOSPC, "no space after terminal rename") + real_fsync_directory(directory) + + monkeypatch.setattr(lanes.os, "replace", replace_then_mark) + monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace) + + lanes.execute_claim("cassandra", "t_enospc") + + pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) + assert terminal_replaced["value"] is True + assert fail_once["value"] is False + assert len(pending) == 1 + assert blocks == [] + assert completions == [] + assert task.status == "running" + assert any("terminal replayable=True" in body for body in comments) + + # A restarted runner sees the exact pending journal and completes the run. + assert lanes.recover_pending_finalizations() == 1 + assert len(completions) == 1 + assert task.status == "done" + assert not pending[0].exists() + committed = list((state_root / "cassandra").glob("*.terminal.committed.json")) + assert len(committed) == 1 + + +def test_terminal_payload_cannot_select_a_different_board( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3) + lanes.atomic_json( + path, + { + "board": "beta", + "task_id": "t_beta", + "expected_run_id": 9, + "result": json.dumps(_completed_result()), + "summary": "forged", + "metadata": {}, + "kanban_state": "pending", + }, + ) + alpha_task = SimpleNamespace( + id="t_alpha", status="running", current_run_id=3, assignee="cli-auto" + ) + opened = [] + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (opened.append(board) or Connection()), + get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None, + reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + + assert opened == ["alpha"] + assert reclaimed == ["t_alpha"] + assert not path.exists() + quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 + + +def test_terminal_finalize_rereads_journal_after_a_path_swap( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("alpha", "t_alpha") + path, safe_record = lanes._write_terminal_record( + state_file, + board="alpha", + task_id="t_alpha", + run_id=4, + structured=_completed_result("safe"), + summary="safe", + metadata={}, + ) + forged = dict(safe_record) + forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7}) + lanes.atomic_json(path, forged) + opened = [] + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")), + ) + + assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign" + assert opened == [] + + +@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"']) +def test_malformed_exact_run_journal_is_quarantined_and_reclaimed( + tmp_path: Path, + monkeypatch, + payload: bytes, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12) + path.parent.mkdir(parents=True) + path.write_bytes(payload) + task = SimpleNamespace( + id="t_partial", status="running", current_run_id=12, assignee="cli-auto" + ) + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + reclaim_task=lambda *_args, **kwargs: (reclaimed.append(kwargs) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + + assert reclaimed == [{ + "reason": ( + "terminal journal recovery failed (malformed-payload); " + "exact run may retry" + ), + "expected_run_id": 12, + }] + assert not path.exists() + assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False + quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 diff --git a/testing/tests/test_hermes_cli_finalization_authority.py b/testing/tests/test_hermes_cli_finalization_authority.py new file mode 100644 index 00000000..c8c26c31 --- /dev/null +++ b/testing/tests/test_hermes_cli_finalization_authority.py @@ -0,0 +1,53 @@ +"""Post-transaction authority classification for exact terminal results.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + SimpleNamespace, + lanes, + nullcontext, + pytest, +) + + +@pytest.mark.parametrize( + ("status", "current_run_id", "expected"), + [ + ("running", 17, "pending"), + ("running", 18, "stale"), + ("ready", None, "stale"), + ("blocked", None, "stale"), + ("triage", None, "stale"), + ], +) +def test_failed_completion_is_pending_only_for_the_exact_live_run( + status: str, + current_run_id: int | None, + expected: str, +): + task = SimpleNamespace( + id="t_authority", + status=status, + current_run_id=current_run_id, + completed_run_id=None, + result=None, + ) + + class Connection: + def close(self): + return None + + database = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=lambda *_args, **_kwargs: False, + ) + + outcome = lanes._finalize_document_db( + database, + lanes.TerminalIdentity("cassandra", "t_authority", 17, "pending"), + {"result": "accepted", "summary": "accepted", "metadata": {}}, + ) + + assert outcome == expected diff --git a/testing/tests/test_hermes_cli_goal_execution.py b/testing/tests/test_hermes_cli_goal_execution.py new file mode 100644 index 00000000..ce4ebc37 --- /dev/null +++ b/testing/tests/test_hermes_cli_goal_execution.py @@ -0,0 +1,151 @@ +"""Structured goal execution and local completion judging.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + lanes, + nullcontext, + pytest, + sys, +) + + +def test_artifact_gc_is_interval_gated( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0) + calls = [] + monkeypatch.setattr( + lanes, "gc_lane_artifacts", lambda **kwargs: (calls.append(kwargs) or 0) + ) + + assert lanes.maybe_gc_lane_artifacts(now=1000.0) == 0 + assert lanes.maybe_gc_lane_artifacts(now=1001.0) == 0 + assert len(calls) == 1 + + +@pytest.mark.parametrize( + ("result", "expected_action"), + [ + (lanes.ProcessResult(0, "plain text only", None, False), "block"), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "findings": [], + "blockers": [], + }, + False, + ), + "complete", + ), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "The full test suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ), + "block", + ), + ], +) +def test_claim_requires_structured_evidence_and_surfaces_artifacts( + tmp_path: Path, + monkeypatch, + result, + expected_action, +): + task = SimpleNamespace( + id="t_worker", + status="running", + result=None, + current_run_id=4, + assignee="cli-auto", + max_runtime_seconds=60, + ) + calls = [] + heartbeats = [] + connections = [] + artifact = tmp_path / "reports/result.json" + artifact.parent.mkdir() + artifact.write_text("{}\n", encoding="utf-8") + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "bounded objective", + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + def run_provider(*args, **_kwargs): + assert connections[0].closed + before_heartbeat = len(connections) + assert args[6]("working") is True + assert len(connections) == before_heartbeat + 1 + assert connections[-1].closed + return result + + monkeypatch.setattr(lanes, "run_provider", run_provider) + + lanes.execute_claim("cassandra", "t_worker") + + assert calls[0][0] == expected_action + assert heartbeats == [("working", 4)] + if expected_action == "complete": + assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] + assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] + else: + assert calls[0][1]["kind"] == "capability" + assert all(connection.closed for connection in connections) diff --git a/testing/tests/test_hermes_cli_lane_routing.py b/testing/tests/test_hermes_cli_lane_routing.py index 14f880fc..49935d54 100644 --- a/testing/tests/test_hermes_cli_lane_routing.py +++ b/testing/tests/test_hermes_cli_lane_routing.py @@ -11,7 +11,7 @@ from pathlib import Path SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" sys.path.insert(0, str(SCRIPTS)) SCRIPT = SCRIPTS / "cli_lane_runner.py" -SPEC = importlib.util.spec_from_file_location("cli_lane_routing", SCRIPT) +SPEC = importlib.util.spec_from_file_location("cli_lane_runner_routing_test", SCRIPT) assert SPEC and SPEC.loader lanes = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = lanes diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py deleted file mode 100644 index 94d9cb58..00000000 --- a/testing/tests/test_hermes_cli_lanes.py +++ /dev/null @@ -1,3239 +0,0 @@ -"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" - -from __future__ import annotations - -import errno -import importlib.util -import hashlib -import json -import os -import signal -import stat -import sys -from contextlib import nullcontext -from pathlib import Path -from types import SimpleNamespace - -import pytest -import yaml - - -SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" -sys.path.insert(0, str(SCRIPTS)) -HERMES = Path(__file__).parents[2] / "services/hermes" -KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" -FLUX_HERMES = ( - Path(__file__).parents[2] - / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" -) - - -def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -lanes = _load("cli_lane_runner") -policy = _load("claude_command_policy") -migration = _load("migrate_herdr_state") -auth_patch = _load("patch_hermes_auth") -tui_gateway_patch = _load("patch_tui_gateway") -codex_runtime_patch = _load("patch_codex_runtime") -ttyd_patch = _load("patch_ttyd_index") -client_config = _load("configure_agent_clients") - - -def _agent_deployment() -> dict: - return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) - - -def _services() -> dict[str, dict]: - return { - item["metadata"]["name"]: item - for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) - if item - } - - -def _oauth_deployment(name: str) -> dict: - documents = [ - item - for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) - if item - ] - return next( - item - for item in documents - if item["kind"] == "Deployment" and item["metadata"]["name"] == name - ) - - -class _SwitchyardResponse: - """Minimal context-managed response used by routing contract tests.""" - - def __init__(self, selected: str, rationale: str = "local classifier vote"): - self.headers = { - "x-model-router-selected-model": selected, - "x-model-router-rationale": rationale, - } - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self): - return b"{}" - - -def test_auto_lane_uses_switchyard_worker_decision(): - observed = {} - - def route(request, timeout): - observed["payload"] = json.loads(request.data) - observed["timeout"] = timeout - return _SwitchyardResponse("worker/claude/claude-opus-5/xhigh") - - route = lanes.select_route( - "Perform the security review.", - "cli-auto", - open_request=route, - ) - - assert route.provider == "claude" - assert route.model == "claude-opus-5" - assert route.effort == "xhigh" - assert route.classifier == "switchyard-classifier" - assert observed["payload"]["model"] == "atlas/worker/auto/maximum" - assert observed["timeout"] == 60 - - -def test_manual_lane_is_still_enforced_by_switchyard(): - observed = {} - - def route(request, timeout): - observed["payload"] = json.loads(request.data) - return _SwitchyardResponse( - "worker/claude/claude-sonnet-5/high", "manual route" - ) - - route = lanes.select_route( - "Implement it.", - "cli-claude-high", - open_request=route, - ) - - assert route.provider == "claude" - assert route.effort == "high" - assert route.classifier == "switchyard-manual" - assert route.reason == "manual route" - assert observed["payload"]["model"] == "atlas/worker/manual/claude/high" - - -def test_cross_provider_retry_passes_failed_provider_to_switchyard(): - observed = {} - - def route(request, timeout): - observed["payload"] = json.loads(request.data) - return _SwitchyardResponse("worker/claude/claude-sonnet-5/high") - - route = lanes.select_route( - "Retry after capacity exhaustion.", - "cli-auto", - exclude_provider="codex", - open_request=route, - ) - - assert route.provider == "claude" - assert route.classifier == "switchyard-classifier" - content = observed["payload"]["messages"][0]["content"] - assert "codex provider failed or exhausted capacity" in content - - -def test_classifier_cannot_select_a_freshly_excluded_provider(): - payloads = [] - - def route(request, timeout): - payload = json.loads(request.data) - payloads.append(payload) - if len(payloads) == 1: - return _SwitchyardResponse("worker/claude/opus/xhigh") - return _SwitchyardResponse("worker/codex/sol/xhigh", "healthy route") - - selected = lanes.select_route( - "Perform a consequential review.", - "cli-auto", - exclude_provider="claude", - exclude_reason="is unavailable according to fresh native health", - open_request=route, - ) - - assert selected.provider == "codex" - assert selected.effort == "xhigh" - assert selected.classifier == "switchyard-classifier-health-guard" - assert payloads[0]["model"] == "atlas/worker/auto/maximum" - assert payloads[1]["model"] == "atlas/worker/manual/codex/xhigh" - assert "claude provider is unavailable" in payloads[0]["messages"][0]["content"] - - -def test_fresh_native_health_excludes_only_one_proven_down_provider( - tmp_path: Path, monkeypatch -): - paths = { - "codex": tmp_path / "codex.json", - "claude": tmp_path / "claude.json", - } - paths["codex"].write_text('{"state":"available"}\n', encoding="utf-8") - paths["claude"].write_text('{"state":"unavailable"}\n', encoding="utf-8") - monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) - - now = max(path.stat().st_mtime for path in paths.values()) - assert lanes.fresh_unavailable_provider(now=now) == "claude" - - paths["codex"].write_text('{"state":"unavailable"}\n', encoding="utf-8") - now = max(path.stat().st_mtime for path in paths.values()) - assert lanes.fresh_unavailable_provider(now=now) is None - - -def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch): - paths = { - "codex": tmp_path / "codex.json", - "claude": tmp_path / "claude.json", - } - paths["codex"].write_text( - '{"state":"available","authenticated":true}\n', encoding="utf-8" - ) - paths["claude"].write_text( - '{"state":"unavailable","authenticated":false}\n', encoding="utf-8" - ) - monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) - - stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60 - assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude" - - paths["claude"].write_text( - '{"state":"unavailable","authenticated":true}\n', encoding="utf-8" - ) - transient_stale = paths["claude"].stat().st_mtime + 10 * 60 - assert lanes.fresh_unavailable_provider(now=transient_stale) is None - - -def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch): - """Kanban workers must not fall back to credential-free persistent homes.""" - monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex") - monkeypatch.setenv("CLAUDE_CONFIG_DIR", "/runtime-access/claude") - - env = lanes._base_env() - - assert env["HOME"] == str(lanes.DATA_ROOT / "home") - assert env["CODEX_HOME"] == "/runtime-access/codex" - assert env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert env["GIT_TERMINAL_PROMPT"] == "0" - - -def test_claude_session_is_reserved_before_first_process(tmp_path: Path, monkeypatch): - state: dict = {} - state_file = tmp_path / "state.json" - observed = {} - - def fake_stream(command, **kwargs): - observed["command"] = command - observed["state"] = json.loads(state_file.read_text()) - return lanes.ProcessResult(0, "", {"status": "completed"}, False) - - monkeypatch.setattr(lanes, "stream_process", fake_stream) - route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) - - lanes.run_provider( - route, - "Do the work.", - tmp_path, - state, - state_file, - tmp_path / "worker.log", - lambda _note: True, - 60, - ) - - reserved = observed["state"]["claude_session_id"] - assert reserved - assert ["--session-id", reserved] == observed["command"][-4:-2] - - -def test_claude_recovery_uses_verified_error_state_machine(tmp_path: Path, monkeypatch): - state = {"claude_session_id": "d4f91f59-88cb-43d0-a64f-f3284688fe9e", "claude_started": True} - state_file = tmp_path / "state.json" - calls = [] - - def fake_stream(command, **kwargs): - calls.append(command) - if len(calls) == 1: - return lanes.ProcessResult( - 1, - "No conversation found with session ID: d4f91f59-88cb-43d0-a64f-f3284688fe9e", - None, - False, - ) - return lanes.ProcessResult(0, "", {"status": "completed"}, False) - - monkeypatch.setattr(lanes, "stream_process", fake_stream) - route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) - lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60) - - assert "--resume" in calls[0] - assert "--session-id" in calls[1] - assert calls[0][calls[0].index("--resume") + 1] == calls[1][calls[1].index("--session-id") + 1] - - -def test_claude_collision_never_double_starts_a_reserved_session(tmp_path: Path, monkeypatch): - state = {"claude_session_id": "24927cee-f3da-4bcf-b624-602c22860155"} - state_file = tmp_path / "state.json" - calls = [] - - def fake_stream(command, **kwargs): - calls.append(command) - if len(calls) == 1: - return lanes.ProcessResult(1, "Session ID already in use", None, False) - return lanes.ProcessResult(0, "", {"status": "completed"}, False) - - monkeypatch.setattr(lanes, "stream_process", fake_stream) - route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) - lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60) - - assert "--session-id" in calls[0] - assert "--resume" in calls[1] - assert calls[0][calls[0].index("--session-id") + 1] == calls[1][calls[1].index("--resume") + 1] - - -def test_codex_thread_started_is_persisted_before_completion(tmp_path: Path): - state: dict = {} - state_file = tmp_path / "state.json" - - lanes._event_payload( - "codex", - json.dumps({"type": "thread.started", "thread_id": "thread-123"}), - state, - state_file, - ) - - assert json.loads(state_file.read_text())["codex_thread_id"] == "thread-123" - - -def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch): - state = {"codex_thread_id": "failed-thread"} - state_file = tmp_path / "state.json" - calls = [] - - def fake_stream(command, **kwargs): - calls.append(command) - if len(calls) == 1: - return lanes.ProcessResult( - 1, - "thread/resume failed: no rollout found for thread id failed-thread", - None, - False, - ) - return lanes.ProcessResult(0, "", {"status": "completed"}, False) - - monkeypatch.setattr(lanes, "stream_process", fake_stream) - route = lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ) - - result = lanes.run_provider( - route, - "Continue.", - tmp_path, - state, - state_file, - tmp_path / "log", - lambda _: True, - 60, - ) - - assert result.returncode == 0 - assert "resume" in calls[0] - assert "resume" not in calls[1] - assert "codex_thread_id" not in json.loads(state_file.read_text()) - - -def test_codex_result_files_are_unique_and_prior_turns_are_retained( - tmp_path: Path, - monkeypatch, -): - state = {"run_id": 17} - state_file = tmp_path / "task.json" - result_paths = [] - - def fake_stream(command, **_kwargs): - result_path = Path(command[command.index("-o") + 1]) - result_path.write_text( - json.dumps( - { - "status": "completed", - "summary": f"turn {len(result_paths) + 1}", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "findings": [], - "blockers": [], - } - ), - encoding="utf-8", - ) - result_paths.append(result_path) - return lanes.ProcessResult(0, "", None, False) - - monkeypatch.setattr(lanes, "stream_process", fake_stream) - route = lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ) - - first = lanes.run_provider( - route, - "First turn.", - tmp_path, - state, - state_file, - tmp_path / "worker.log", - lambda _note: True, - 60, - ) - second = lanes.run_provider( - route, - "Second turn.", - tmp_path, - state, - state_file, - tmp_path / "worker.log", - lambda _note: True, - 60, - ) - - assert first.structured["summary"] == "turn 1" - assert second.structured["summary"] == "turn 2" - assert len(set(result_paths)) == 2 - assert all(path.exists() for path in result_paths) - assert all(path.stat().st_mode & 0o777 == 0o600 for path in result_paths) - - -def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path): - result = lanes.stream_process( - [sys.executable, "-c", "print('authentication work completed')"], - provider="codex", - cwd=tmp_path, - env=dict(os.environ), - log_path=tmp_path / "worker.log", - state={}, - state_file=tmp_path / "state.json", - heartbeat=lambda _note: True, - max_runtime=60, - ) - - assert result.returncode == 0 - assert result.capacity_failure is False - - -def test_worker_process_uses_isolated_process_session(monkeypatch, tmp_path: Path): - original_popen = lanes.subprocess.Popen - calls = [] - - def recording_popen(*args, **kwargs): - calls.append(kwargs) - return original_popen(*args, **kwargs) - - monkeypatch.setattr(lanes.subprocess, "Popen", recording_popen) - result = lanes.stream_process( - [sys.executable, "-c", "print('done')"], - provider="codex", - cwd=tmp_path, - env=dict(os.environ), - log_path=tmp_path / "worker.log", - state={}, - state_file=tmp_path / "state.json", - heartbeat=lambda _note: True, - max_runtime=60, - ) - - assert result.returncode == 0 - assert calls[0]["start_new_session"] is True - - -def test_worker_process_group_is_killed_after_leader_exits(monkeypatch): - class FinishedProcess: - pid = 4321 - - @staticmethod - def poll(): - return 0 - - signals = [] - monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig))) - - lanes._terminate_worker_process(FinishedProcess()) - - assert signals == [(4321, signal.SIGKILL)] - - -def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): - class RunningProcess: - pid = 4321 - running = True - - @classmethod - def poll(cls): - return None if cls.running else 0 - - @classmethod - def wait(cls, timeout): - assert timeout == 10 - cls.running = False - return 0 - - group_signals = [] - process_signals = [] - monkeypatch.setattr( - lanes, - "_descendant_processes", - lambda _pid: {5000: (5000, 12345)}, - ) - monkeypatch.setattr(lanes, "_process_identity_matches", lambda _pid, _start: True) - monkeypatch.setattr( - lanes.os, - "killpg", - lambda process_group, sig: group_signals.append((process_group, sig)), - ) - monkeypatch.setattr( - lanes.os, - "kill", - lambda pid, sig: process_signals.append((pid, sig)), - ) - - lanes._terminate_worker_process(RunningProcess()) - - assert set(group_signals) == { - (4321, signal.SIGTERM), - (5000, signal.SIGTERM), - (4321, signal.SIGKILL), - (5000, signal.SIGKILL), - } - assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -def test_accepted_result_survives_failed_finalization_and_replays_exact_run( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - task = SimpleNamespace( - id="t_terminal", - status="running", - result=None, - current_run_id=41, - assignee="cli-auto", - max_runtime_seconds=60, - ) - completions = [] - comments = [] - permit_completion = {"value": False} - - class Connection: - def close(self): - return None - - def complete_task(_conn, _task_id, **kwargs): - completions.append(kwargs) - if not permit_completion["value"]: - return False - task.status = "done" - task.result = kwargs["result"] - task.current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Finish and verify the objective.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=complete_task, - block_task=lambda *_args, **_kwargs: pytest.fail( - "an accepted journaled result must not be converted into a block" - ), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - structured = { - "status": "completed", - "summary": "Verified exact terminal result.", - "changed_files": [], - "tests_run": ["pytest: passed"], - "artifacts": [], - "findings": [], - "blockers": [], - } - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False), - ) - - lanes.execute_claim("cassandra", "t_terminal") - - candidates = list((state_root / "cassandra").glob("*.candidate-*.json")) - terminals = list((state_root / "cassandra").glob("*.terminal.pending.json")) - assert len(candidates) == 1 - assert json.loads(candidates[0].read_text())["structured"] == structured - assert len(terminals) == 1 - assert json.loads(terminals[0].read_text())["kanban_state"] == "pending" - assert task.status == "running" - assert any("durably journaled" in body for body in comments) - - permit_completion["value"] = True - assert lanes.recover_pending_finalizations() == 1 - assert task.status == "done" - assert len(completions) == 2 - assert not terminals[0].exists() - committed = list( - (state_root / "cassandra").glob("*.terminal.committed.json") - ) - assert len(committed) == 1 - terminal = json.loads(committed[0].read_text()) - assert terminal["kanban_state"] == "committed" - assert completions[-1]["result"] == terminal["result"] - - -def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - state_file = lanes.state_path("cassandra", "t_terminal") - terminal_path, _record = lanes._write_terminal_record( - state_file, - board="cassandra", - task_id="t_terminal", - run_id=8, - structured={ - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "findings": [], - "blockers": [], - }, - summary="done", - metadata={}, - ) - assert terminal_path.exists() - task = SimpleNamespace( - id="t_terminal", - status="running", - assignee="cli-auto", - current_run_id=8, - ) - reclaimed = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - list_tasks=lambda _conn: [task], - reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) - - lanes.recover_orphans() - - assert reclaimed == [] - - -def test_terminal_replay_never_crosses_into_a_replacement_run( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - state_file = lanes.state_path("cassandra", "t_terminal") - terminal_path, _record = lanes._write_terminal_record( - state_file, - board="cassandra", - task_id="t_terminal", - run_id=8, - structured={ - "status": "completed", - "summary": "old run", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "findings": [], - "blockers": [], - }, - summary="old run", - metadata={}, - ) - replacement = SimpleNamespace( - id="t_terminal", - status="running", - result=None, - current_run_id=9, - ) - completions = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: replacement, - complete_task=lambda *_args, **_kwargs: completions.append(True), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 0 - assert completions == [] - assert not terminal_path.exists() - quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) - assert len(quarantined) == 1 - - -def _completed_result(summary: str = "done") -> dict: - return { - "status": "completed", - "summary": summary, - "changed_files": [], - "tests_run": [], - "artifacts": [], - "findings": [], - "blockers": [], - } - - -def test_terminal_recovery_can_complete_the_exact_latest_ended_run( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - state_file = lanes.state_path("cassandra", "t_ended") - pending, _record = lanes._write_terminal_record( - state_file, - board="cassandra", - task_id="t_ended", - run_id=17, - structured=_completed_result("accepted before legacy block"), - summary="accepted before legacy block", - metadata={}, - ) - task = SimpleNamespace( - id="t_ended", - status="blocked", - result=None, - current_run_id=None, - assignee="cli-auto", - ) - guards = [] - - class Connection: - def close(self): - return None - - def complete_task(_conn, _task_id, **kwargs): - guards.append(kwargs) - task.status = "done" - task.result = kwargs["result"] - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - complete_task=complete_task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 1 - assert guards[0]["replay_ended_run_id"] == 17 - assert "expected_run_id" not in guards[0] - assert task.status == "done" - assert not pending.exists() - - -def test_complete_exception_after_journal_is_replayable_and_never_blocks( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - task = SimpleNamespace( - id="t_crash", - status="running", - result=None, - current_run_id=51, - assignee="cli-auto", - max_runtime_seconds=60, - ) - completion_raises = {"value": True} - blocks = [] - comments = [] - - class Connection: - def close(self): - return None - - def complete_task(_conn, _task_id, **kwargs): - if completion_raises["value"]: - raise RuntimeError("crash between journal and commit") - task.status = "done" - task.result = kwargs["result"] - task.current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Finish safely.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=complete_task, - block_task=lambda *_args, **kwargs: blocks.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: lanes.ProcessResult( - 0, "", _completed_result("crash-safe result"), False - ), - ) - - lanes.execute_claim("cassandra", "t_crash") - - pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) - assert len(pending) == 1 - assert blocks == [] - assert any("RuntimeError" in body for body in comments) - - completion_raises["value"] = False - assert lanes.recover_pending_finalizations() == 1 - assert task.status == "done" - assert not pending[0].exists() - - -def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - task = SimpleNamespace( - id="t_enospc", - status="running", - result=None, - current_run_id=52, - assignee="cli-auto", - max_runtime_seconds=60, - ) - blocks = [] - comments = [] - completions = [] - - class Connection: - def close(self): - return None - - def complete_task(_conn, _task_id, **kwargs): - completions.append(kwargs) - task.status = "done" - task.result = kwargs["result"] - task.current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Finish without losing the result.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=complete_task, - block_task=lambda *_args, **kwargs: blocks.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: lanes.ProcessResult( - 0, "", _completed_result("persisted before ENOSPC"), False - ), - ) - real_replace = lanes.os.replace - real_fsync_directory = lanes._fsync_directory - terminal_replaced = {"value": False} - fail_once = {"value": True} - - def replace_then_mark(source, destination): - real_replace(source, destination) - if str(destination).endswith(".terminal.pending.json"): - terminal_replaced["value"] = True - - def fail_after_terminal_replace(directory): - if terminal_replaced["value"] and fail_once["value"]: - fail_once["value"] = False - raise OSError(errno.ENOSPC, "no space after terminal rename") - real_fsync_directory(directory) - - monkeypatch.setattr(lanes.os, "replace", replace_then_mark) - monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace) - - lanes.execute_claim("cassandra", "t_enospc") - - pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) - assert terminal_replaced["value"] is True - assert fail_once["value"] is False - assert len(pending) == 1 - assert blocks == [] - assert completions == [] - assert task.status == "running" - assert any("terminal replayable=True" in body for body in comments) - - # A restarted runner sees the exact pending journal and completes the run. - assert lanes.recover_pending_finalizations() == 1 - assert len(completions) == 1 - assert task.status == "done" - assert not pending[0].exists() - committed = list((state_root / "cassandra").glob("*.terminal.committed.json")) - assert len(committed) == 1 - - -def test_terminal_payload_cannot_select_a_different_board( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3) - lanes.atomic_json( - path, - { - "board": "beta", - "task_id": "t_beta", - "expected_run_id": 9, - "result": json.dumps(_completed_result()), - "summary": "forged", - "metadata": {}, - "kanban_state": "pending", - }, - ) - alpha_task = SimpleNamespace( - id="t_alpha", status="running", current_run_id=3, assignee="cli-auto" - ) - opened = [] - reclaimed = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: (opened.append(board) or Connection()), - get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None, - reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 0 - - assert opened == ["alpha"] - assert reclaimed == ["t_alpha"] - assert not path.exists() - quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine")) - assert len(quarantined) == 1 - assert quarantined[0].stat().st_mode & 0o777 == 0o600 - - -def test_terminal_finalize_rereads_journal_after_a_path_swap( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - state_file = lanes.state_path("alpha", "t_alpha") - path, safe_record = lanes._write_terminal_record( - state_file, - board="alpha", - task_id="t_alpha", - run_id=4, - structured=_completed_result("safe"), - summary="safe", - metadata={}, - ) - forged = dict(safe_record) - forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7}) - lanes.atomic_json(path, forged) - opened = [] - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")), - ) - - assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign" - assert opened == [] - - -@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"']) -def test_malformed_exact_run_journal_is_quarantined_and_reclaimed( - tmp_path: Path, - monkeypatch, - payload: bytes, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12) - path.parent.mkdir(parents=True) - path.write_bytes(payload) - task = SimpleNamespace( - id="t_partial", status="running", current_run_id=12, assignee="cli-auto" - ) - reclaimed = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 0 - - assert reclaimed == [True] - assert not path.exists() - assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False - quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) - assert len(quarantined) == 1 - assert quarantined[0].stat().st_mode & 0o777 == 0o600 - - -@pytest.mark.parametrize( - "structured", - [ - {"status": "completed", "blockers": {}}, - {**_completed_result(), "changed_files": "src/a.py"}, - {**_completed_result(), "unexpected": True}, - {**_completed_result(), "blockers": ["work remains"]}, - {**_completed_result("tests are still running")}, - ], -) -def test_terminal_record_requires_exact_completed_result_contract(structured): - record = { - "board": "cassandra", - "task_id": "t_schema", - "expected_run_id": 1, - "result": json.dumps(structured), - "summary": str(structured.get("summary") or "done"), - "metadata": {}, - "kanban_state": "pending", - } - - assert lanes._terminal_record_valid(record) is False - - -def test_minimal_completed_result_with_mapping_blockers_quarantines_without_completion( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - path = lanes._terminal_path(lanes.state_path("cassandra", "t_schema"), 6) - lanes.atomic_json( - path, - { - "board": "cassandra", - "task_id": "t_schema", - "expected_run_id": 6, - "result": json.dumps({"status": "completed", "blockers": {}}), - "summary": "done", - "metadata": {}, - "kanban_state": "pending", - }, - ) - task = SimpleNamespace( - id="t_schema", status="running", current_run_id=6, assignee="cli-auto" - ) - completed = [] - reclaimed = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - complete_task=lambda *_args, **_kwargs: completed.append(True), - reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 0 - assert completed == [] - assert reclaimed == [True] - assert not path.exists() - quarantined = list((path.parent / "quarantine").glob("*.quarantine")) - assert len(quarantined) == 1 - assert quarantined[0].stat().st_mode & 0o777 == 0o600 - - -def test_quarantine_avoids_symlink_and_mode_collision_destinations( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - path = lanes._terminal_path(lanes.state_path("cassandra", "t_collision"), 2) - path.parent.mkdir(parents=True) - path.write_bytes(b"invalid") - quarantine = path.parent / "quarantine" - quarantine.mkdir() - fingerprint = hashlib.sha256(b"invalid").hexdigest()[:16] - base = f"{path.name}.malformed-payload.{fingerprint}" - victim = tmp_path / "victim" - victim.write_text("unchanged", encoding="utf-8") - (quarantine / f"{base}.0.quarantine").symlink_to(victim) - collision = quarantine / f"{base}.1.quarantine" - collision.write_text("attacker collision", encoding="utf-8") - collision.chmod(0o644) - monkeypatch.setattr( - lanes.os, - "chmod", - lambda *_args, **_kwargs: pytest.fail("quarantine must not chmod foreign inodes"), - ) - - destination = lanes._quarantine_terminal( - path, - lanes._terminal_identity(path), - "malformed-payload", - ) - - assert not path.exists() - assert destination.name == f"{base}.2.quarantine" - assert destination.stat().st_mode & 0o777 == 0o600 - diagnostic = json.loads(destination.read_text(encoding="utf-8")) - assert diagnostic["size"] == len(b"invalid") - assert diagnostic["sha256"] == hashlib.sha256(b"invalid").hexdigest() - assert diagnostic["source_kind"] == "regular" - assert victim.read_text(encoding="utf-8") == "unchanged" - assert collision.read_text(encoding="utf-8") == "attacker collision" - assert collision.stat().st_mode & 0o777 == 0o644 - - -def test_hardlinked_terminal_source_is_never_chmodded_or_copied_as_data( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - path = lanes._terminal_path(lanes.state_path("cassandra", "t_hardlink"), 5) - path.parent.mkdir(parents=True) - foreign = tmp_path / "foreign" - foreign.write_text("foreign inode contents", encoding="utf-8") - foreign.chmod(0o644) - os.link(foreign, path) - monkeypatch.setattr( - lanes.os, - "chmod", - lambda *_args, **_kwargs: pytest.fail("hardlinked source must not be chmodded"), - ) - - destination = lanes._quarantine_terminal( - path, - lanes._terminal_identity(path), - "malformed-payload", - ) - - assert not path.exists() - assert foreign.read_text(encoding="utf-8") == "foreign inode contents" - assert foreign.stat().st_mode & 0o777 == 0o644 - assert destination.stat().st_mode & 0o777 == 0o600 - diagnostic = json.loads(destination.read_text(encoding="utf-8")) - assert diagnostic["source_kind"] == "hardlink" - assert "foreign inode contents" not in destination.read_text(encoding="utf-8") - - -def test_quarantine_preserves_and_replays_an_atomic_replacement( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - path = lanes._terminal_path(lanes.state_path("cassandra", "t_swap"), 21) - path.parent.mkdir(parents=True) - path.write_bytes(b"malformed") - identity = lanes._terminal_identity(path) - assert identity is not None - structured = _completed_result("replacement result") - replacement_record = { - "board": "cassandra", - "task_id": "t_swap", - "expected_run_id": 21, - "result": json.dumps(structured, sort_keys=True), - "summary": structured["summary"], - "metadata": {}, - "kanban_state": "pending", - "recorded_at": lanes.utc_now(), - } - replacement = path.with_name("replacement.tmp") - replacement.write_text( - json.dumps(replacement_record, sort_keys=True), - encoding="utf-8", - ) - replacement.chmod(0o600) - real_fsync = lanes.os.fsync - swapped = {"value": False} - - def swap_on_quarantine_directory_fsync(descriptor): - descriptor_stat = os.fstat(descriptor) - if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]: - os.replace(replacement, path) - swapped["value"] = True - real_fsync(descriptor) - - monkeypatch.setattr(lanes.os, "fsync", swap_on_quarantine_directory_fsync) - - lanes._quarantine_terminal(path, identity, "malformed-payload") - - assert swapped["value"] is True - assert path.exists() - assert lanes._terminal_record_valid( - lanes._load_terminal_json(path, identity), identity - ) - - task = SimpleNamespace( - id="t_swap", status="running", current_run_id=21, result=None, assignee="cli-auto" - ) - - class Connection: - def close(self): - return None - - def complete_task(_conn, _task_id, **kwargs): - task.status = "done" - task.result = kwargs["result"] - task.current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - complete_task=complete_task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 1 - assert task.status == "done" - assert not path.exists() - - -def test_invalid_utf8_journal_quarantines_and_does_not_stop_later_replay( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - bad = lanes._terminal_path(lanes.state_path("cassandra", "a_bad_utf8"), 31) - bad.parent.mkdir(parents=True) - bad.write_bytes(b"\xff\xfe\x80not-json") - good, _record = lanes._write_terminal_record( - lanes.state_path("cassandra", "z_good"), - board="cassandra", - task_id="z_good", - run_id=32, - structured=_completed_result("valid after invalid UTF-8"), - summary="valid after invalid UTF-8", - metadata={}, - ) - tasks = { - "a_bad_utf8": SimpleNamespace( - id="a_bad_utf8", - status="running", - current_run_id=31, - result=None, - assignee="cli-auto", - ), - "z_good": SimpleNamespace( - id="z_good", - status="running", - current_run_id=32, - result=None, - assignee="cli-auto", - ), - } - - class Connection: - def close(self): - return None - - def reclaim_task(_conn, task_id, **_kwargs): - tasks[task_id].status = "ready" - tasks[task_id].current_run_id = None - return True - - def complete_task(_conn, task_id, **kwargs): - tasks[task_id].status = "done" - tasks[task_id].result = kwargs["result"] - tasks[task_id].current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, task_id: tasks[task_id], - reclaim_task=reclaim_task, - complete_task=complete_task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 1 - assert not bad.exists() - assert not good.exists() - assert tasks["a_bad_utf8"].status == "ready" - assert tasks["z_good"].status == "done" - diagnostic_path = next((bad.parent / "quarantine").glob("*.quarantine")) - diagnostic = json.loads(diagnostic_path.read_text(encoding="utf-8")) - assert diagnostic["size"] == len(b"\xff\xfe\x80not-json") - assert diagnostic_path.stat().st_size < 4096 - - -@pytest.mark.parametrize( - "size", - [lanes.MAX_TERMINAL_RECORD_BYTES + 1, 16 * 1024 * 1024], -) -def test_oversized_sparse_journal_has_bounded_read_and_quarantine( - tmp_path: Path, - monkeypatch, - size: int, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - huge = lanes._terminal_path(lanes.state_path("cassandra", "a_huge"), 41) - huge.parent.mkdir(parents=True) - with huge.open("wb") as stream: - stream.seek(size - 1) - stream.write(b"\0") - good, _record = lanes._write_terminal_record( - lanes.state_path("cassandra", "z_after_huge"), - board="cassandra", - task_id="z_after_huge", - run_id=42, - structured=_completed_result("valid after oversized journal"), - summary="valid after oversized journal", - metadata={}, - ) - tasks = { - "a_huge": SimpleNamespace( - id="a_huge", - status="running", - current_run_id=41, - result=None, - assignee="cli-auto", - ), - "z_after_huge": SimpleNamespace( - id="z_after_huge", - status="running", - current_run_id=42, - result=None, - assignee="cli-auto", - ), - } - reads = [] - real_read_bounded = lanes._read_bounded - - def observe_read_limit(descriptor, limit): - reads.append((os.fstat(descriptor).st_size, limit)) - return real_read_bounded(descriptor, limit) - - monkeypatch.setattr(lanes, "_read_bounded", observe_read_limit) - - class Connection: - def close(self): - return None - - def reclaim_task(_conn, task_id, **_kwargs): - tasks[task_id].status = "ready" - tasks[task_id].current_run_id = None - return True - - def complete_task(_conn, task_id, **kwargs): - tasks[task_id].status = "done" - tasks[task_id].result = kwargs["result"] - tasks[task_id].current_run_id = None - return True - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, task_id: tasks[task_id], - reclaim_task=reclaim_task, - complete_task=complete_task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.recover_pending_finalizations() == 1 - assert not huge.exists() - assert not good.exists() - assert tasks["a_huge"].status == "ready" - assert tasks["z_after_huge"].status == "done" - diagnostics = list((huge.parent / "quarantine").glob("*.quarantine")) - assert len(diagnostics) == 1 - assert diagnostics[0].stat().st_size < 4096 - diagnostic = json.loads(diagnostics[0].read_text(encoding="utf-8")) - assert diagnostic["size"] == size - assert diagnostic["hashed_bytes"] == lanes.QUARANTINE_HASH_BYTES - assert diagnostic["hash_complete"] is False - assert (size, lanes.QUARANTINE_HASH_BYTES) in reads - assert (size, lanes.MAX_TERMINAL_RECORD_BYTES + 1) not in reads - assert max(limit for _file_size, limit in reads) <= ( - lanes.MAX_TERMINAL_RECORD_BYTES + 1 - ) - - -def test_recovery_does_not_parse_committed_journals_every_tick( - tmp_path: Path, - monkeypatch, -): - state_root = tmp_path / "cli-lanes" - monkeypatch.setattr(lanes, "STATE_ROOT", state_root) - committed = lanes._terminal_path( - lanes.state_path("cassandra", "t_done"), 1, "committed" - ) - lanes.atomic_json(committed, {"kanban_state": "committed"}) - reads = [] - monkeypatch.setattr(lanes, "load_json", lambda path: (reads.append(path) or {})) - monkeypatch.setitem( - sys.modules, - "hermes_cli", - SimpleNamespace(kanban_db=SimpleNamespace()), - ) - - assert lanes.recover_pending_finalizations() == 0 - assert reads == [] - - -def test_atomic_json_fsyncs_file_and_directory_and_uses_private_mode( - tmp_path: Path, - monkeypatch, -): - calls = [] - real_fsync = lanes.os.fsync - monkeypatch.setattr( - lanes.os, - "fsync", - lambda descriptor: (calls.append(descriptor), real_fsync(descriptor))[1], - ) - path = tmp_path / "state.json" - - lanes.atomic_json(path, {"safe": True}) - - assert json.loads(path.read_text()) == {"safe": True} - assert path.stat().st_mode & 0o777 == 0o600 - assert len(calls) >= 2 - - -def test_atomic_json_refuses_a_precreated_temp_symlink( - tmp_path: Path, - monkeypatch, -): - path = tmp_path / "state.json" - victim = tmp_path / "credential" - victim.write_text("do not overwrite", encoding="utf-8") - temporary = tmp_path / ".state.json.fixed.tmp" - temporary.symlink_to(victim) - monkeypatch.setattr(lanes.uuid, "uuid4", lambda: SimpleNamespace(hex="fixed")) - - with pytest.raises(FileExistsError): - lanes.atomic_json(path, {"unsafe": True}) - - assert victim.read_text(encoding="utf-8") == "do not overwrite" - assert not path.exists() - - -def test_artifact_gc_prunes_by_age_without_touching_pending_journals( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - board = lanes.STATE_ROOT / "cassandra" - board.mkdir(parents=True) - quarantine = board / "quarantine" - quarantine.mkdir() - old_artifacts = [ - board / "t.run-1.candidate-1.json", - board / "t.run-1.provider-1.result.json", - board / "t.run-1.terminal.committed.json", - quarantine / "t.invalid.1234.quarantine", - ] - for artifact in old_artifacts: - artifact.write_text("{}", encoding="utf-8") - quarantine_symlink = quarantine / "attacker.quarantine" - quarantine_symlink.symlink_to(tmp_path / "missing-target") - pending = board / "t.run-1.terminal.pending.json" - pending.write_text("{}", encoding="utf-8") - old_time = 100.0 - for artifact in old_artifacts: - os.utime(artifact, (old_time, old_time)) - os.utime(pending, (old_time, old_time)) - - assert lanes.gc_lane_artifacts( - now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 - ) == len(old_artifacts) + 1 - assert not any(artifact.exists() for artifact in old_artifacts) - assert not quarantine_symlink.is_symlink() - assert pending.exists() - - -def test_artifact_gc_prunes_oldest_by_count_and_total_bytes( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - board = lanes.STATE_ROOT / "cassandra" - board.mkdir(parents=True) - files = [] - for sequence in range(3): - path = board / f"t.run-1.provider-{sequence}.result.json" - path.write_text("x" * 40, encoding="utf-8") - os.utime(path, (100 + sequence, 100 + sequence)) - files.append(path) - - assert lanes.gc_lane_artifacts( - now=200.0, max_age_seconds=1000, max_count=2, max_bytes=45 - ) == 2 - assert [path.exists() for path in files] == [False, False, True] - - -def test_artifact_gc_preserves_a_concurrent_replacement( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - board = lanes.STATE_ROOT / "cassandra" - board.mkdir(parents=True) - path = board / "t.run-1.provider-1.result.json" - path.write_text("old", encoding="utf-8") - os.utime(path, (100, 100)) - replacement = board / "replacement.tmp" - replacement.write_text("new", encoding="utf-8") - real_unlink = lanes._unlink_artifact_if_same - swapped = {"value": False} - - def swap_before_identity_checked(candidate, observed): - if not swapped["value"]: - os.replace(replacement, candidate) - swapped["value"] = True - return real_unlink(candidate, observed) - - monkeypatch.setattr(lanes, "_unlink_artifact_if_same", swap_before_identity_checked) - - assert lanes.gc_lane_artifacts( - now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 - ) == 0 - assert swapped["value"] is True - assert path.read_text(encoding="utf-8") == "new" - - -def test_artifact_gc_is_interval_gated( - tmp_path: Path, - monkeypatch, -): - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0) - calls = [] - monkeypatch.setattr( - lanes, "gc_lane_artifacts", lambda **kwargs: (calls.append(kwargs) or 0) - ) - - assert lanes.maybe_gc_lane_artifacts(now=1000.0) == 0 - assert lanes.maybe_gc_lane_artifacts(now=1001.0) == 0 - assert len(calls) == 1 - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "findings": [], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "findings": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - status="running", - result=None, - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: ( - calls.append(("complete", kwargs)) or True - ), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - status="running", - result=None, - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: ( - calls.append(("complete", kwargs)) or True - ), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "findings": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "findings": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) - assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - candidates = sorted( - (lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json") - ) - assert len(candidates) == 2 - assert json.loads(candidates[0].read_text())["structured"]["summary"] == ( - "Focused tests passed." - ) - assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith( - "Full tests passed" - ) - assert reports == [] - - -def test_capacity_fallback_preserves_first_claude_structured_response( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_fallback", - status="running", - result=None, - current_run_id=14, - assignee="cli-auto", - max_runtime_seconds=60, - ) - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Complete with a fallback if needed.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda *_args: None, - complete_task=lambda _conn, _task_id, **kwargs: ( - setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True - ), - block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - claude = lanes.Route( - "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, () - ) - codex = lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () - ) - monkeypatch.setattr( - lanes, - "select_route", - lambda _prompt, assignee, **_kwargs: codex - if assignee == "cli-codex-high" - else claude, - ) - reports = [ - lanes.ProcessResult( - 1, - "subscription capacity exhausted", - { - **_completed_result("Claude preserved evidence"), - "status": "incomplete", - "blockers": ["subscription capacity exhausted"], - }, - True, - ), - lanes.ProcessResult(0, "", _completed_result("Codex completed"), False), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - - lanes.execute_claim("cassandra", "t_fallback") - - candidates = sorted( - (lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json") - ) - assert len(candidates) == 2 - first, second = [json.loads(path.read_text()) for path in candidates] - assert (first["provider"], first["structured"]["summary"]) == ( - "claude", - "Claude preserved evidence", - ) - assert (second["provider"], second["structured"]["summary"]) == ( - "codex", - "Codex completed", - ) - assert task.status == "done" - - -def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): - task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") - calls = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( - ValueError("not a Git repository") - ), - block_task=lambda *_args, **kwargs: calls.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - - lanes.execute_claim("cassandra", "t_bad_worktree") - - assert calls[0]["kind"] == "capability" - assert calls[0]["expected_run_id"] == 7 - assert "not a Git repository" in calls[0]["reason"] - - -def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): - workspace = tmp_path / "workspace" - workspace.mkdir() - inside = workspace / "report.json" - outside = tmp_path / "auth.json" - inside.write_text("{}\n", encoding="utf-8") - outside.write_text("secret\n", encoding="utf-8") - - assert lanes.workspace_artifacts( - workspace, - ["report.json", str(outside), "missing.json"], - ) == [str(inside)] - - -def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_resume", - status="running", - result=None, - current_run_id=9, - assignee="cli-auto", - max_runtime_seconds=60, - ) - monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") - state_file = lanes.state_path("cassandra", "t_resume") - state_file.parent.mkdir(parents=True) - state_file.write_text( - json.dumps({"current_route": {"provider": "claude"}}), - encoding="utf-8", - ) - (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") - prompts = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "resume objective", - add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: True, - block_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "git_handoff", - lambda _workspace, output: f"HANDOFF:{output}", - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda _route, prompt, *_args, **_kwargs: ( - prompts.append(prompt) - or lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "findings": [], - "blockers": [], - }, - False, - ) - ), - ) - - lanes.execute_claim("cassandra", "t_resume") - - assert "HANDOFF:prior provider evidence" in prompts[0] - - -def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): - route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) - command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") - assert "--dangerously-bypass-approvals-and-sandbox" in command - assert "--json" in command - assert "--output-schema" in command - assert 'model_reasoning_effort="xhigh"' in command - - claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} - claude = lanes._claude_command( - lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), - "Review.", - claude_state, - False, - ) - assert "--dangerously-skip-permissions" in claude - assert "--output-format" in claude and "stream-json" in claude - assert "--json-schema" in claude - assert "--disallowedTools" in claude - assert "Bash(kubectl apply *)" not in claude - assert "Bash(flux reconcile *)" not in claude - assert "max" not in claude - - -def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): - prompt = lanes.build_prompt("Review the change.", tmp_path) - - assert "put defects and risks in findings" in prompt - assert "blockers array must be empty whenever status is completed" in prompt - assert "findings" in lanes.RESULT_SCHEMA["properties"] - assert set(lanes.RESULT_SCHEMA["required"]) == set( - lanes.RESULT_SCHEMA["properties"] - ) - assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] - - -@pytest.mark.parametrize( - "command", - [ - "git push --force origin main", - "git reset --hard HEAD~1", - "git clean -fd", - ], -) -def test_claude_pretool_hook_blocks_hard_denies(command: str): - assert policy.denial_reason(command) - - -def test_claude_pretool_hook_allows_normal_engineering(): - assert policy.denial_reason("pytest -q testing/tests") is None - assert policy.denial_reason("git push origin feature/hermes") is None - assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None - assert policy.denial_reason("flux reconcile kustomization hermes") is None - assert policy.denial_reason("vault kv get kv/atlas/hermes") is None - - -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): - state = tmp_path / ".claude.json" - settings = tmp_path / "settings.json" - state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") - settings.write_text( - json.dumps( - { - "theme": "dark", - "permissions": { - "deny": [ - "Bash(kubectl apply *)", - "Bash(flux reconcile *)", - "Bash(vault kv *)", - "Bash(custom-owner-rule *)", - ] - }, - } - ) - + "\n", - encoding="utf-8", - ) - - client_config.configure_claude_state(state) - client_config.configure_claude_settings(settings) - - state_value = json.loads(state.read_text()) - settings_value = json.loads(settings.read_text()) - assert state_value["promptQueueUseCount"] == 4 - assert state_value["bypassPermissionsModeAccepted"] is True - assert settings_value["theme"] == "dark" - assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] - assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] - assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] - assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] - assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] - hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] - assert "claude_command_policy.py" in hook["command"] - - -def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): - session = tmp_path / "home/.config/herdr/session.json" - session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") - binary = tmp_path / "tools/bin/herdr" - binary.parent.mkdir(parents=True) - binary.write_text("legacy", encoding="utf-8") - (tmp_path / "home/.claude").mkdir() - (tmp_path / "home/.codex").mkdir() - - archive = migration.archive_legacy_state(tmp_path) - - value = json.loads(archive.read_text()) - assert value["legacy_session"]["agents"][0]["session_id"] == "abc" - assert (tmp_path / "home/.claude").is_dir() - assert (tmp_path / "home/.codex").is_dir() - assert not binary.exists() - assert not (tmp_path / "home/.config/herdr").exists() - - -def test_agent_uses_one_native_kanban_control_plane(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/balanced", - "model": "atlas/auto/balanced", - } - assert config["kanban"]["dispatch_in_gateway"] is True - assert config["kanban"]["default_assignee"] == "cli-auto" - assert config["plugins"]["enabled"] == ["auto-router"] - - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - assert pod["enableServiceLinks"] is False - names = {item["name"] for item in pod["containers"]} - assert "cli-lane-runner" in names - assert "terminal" in names - assert not any("herdr" in name for name in names) - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert "herdr server" not in rendered - assert "herdr-dispatch" not in rendered - - -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): - deployment = _agent_deployment() - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - lane = containers["cli-lane-runner"] - environment = {item["name"]: item["value"] for item in lane["env"]} - - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" - assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, - } - - -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ - pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( - item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] - if item["key"] == "kubernetes.io/hostname" - ) - - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) - assert hermes["resources"]["requests"]["cpu"] == "300m" - - -def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - containers = {item["name"]: item for item in pod["containers"]} - assert "webui" not in containers - - assert "dashboard" not in containers - hermes = containers["hermes"] - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" - assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" - assert hermes["command"] == ["/bin/sh", "-ec"] - startup = hermes["args"][0] - assert ". /opt/data/.env" in startup - assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_DASHBOARD"] == "1" - assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" - assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" - assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - assert hermes["securityContext"]["runAsUser"] == 0 - assert hermes["securityContext"]["runAsGroup"] == 0 - for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): - probe = hermes[probe_name] - assert probe["exec"]["command"] == [ - "curl", - "-fsS", - "http://127.0.0.1:9119/api/status", - ] - - terminal = containers["terminal"] - command = terminal["args"][0] - assert "--base-path /terminal" in command - assert "--check-origin" not in command - assert "/usr/bin/tmux new-session -A" in command - assert "--continue" in command - assert "--yolo" in command - terminal_env = {item["name"]: item["value"] for item in terminal["env"]} - assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - - claude_broker = containers["claude-broker"] - claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} - assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" - assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} - assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} - - args = containers["oauth2-proxy"]["args"] - terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" - dashboard_upstream = "--upstream=http://127.0.0.1:9119/" - assert terminal_upstream in args - assert dashboard_upstream in args - assert args.index(terminal_upstream) < args.index(dashboard_upstream) - assert "--pass-host-header=false" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" - ) - assert patch_init["command"][-1] == "/patched/server.py" - for name in ("hermes", "terminal"): - mounts = containers[name]["volumeMounts"] - assert { - "name": "tui-gateway-patch", - "mountPath": "/opt/hermes/tui_gateway/server.py", - "subPath": "server.py", - } in mounts - - ingress_documents = [ - item - for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) - if item - ] - middlewares = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Middleware" - } - assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ - "replacement" - ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" - ingresses = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Ingress" - } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" - - -def test_broker_services_survive_sibling_container_readiness_loss(): - services = _services() - - for name in ( - "hermes-image-broker", - "hermes-codex-broker", - "hermes-local-image", - "hermes-claude-broker", - ): - assert services[name]["spec"]["publishNotReadyAddresses"] is True - - -def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") - assert "eventsRetryAttempt.current" in dockerfile - assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile - assert "events feed rejected (${ev.code}) — reload the page" in dockerfile - assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile - assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile - assert dockerfile.count("' await api.getSessions(1, 0,") == 2 - assert ".then(() => gw.connect())" in dockerfile - assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile - assert "dashboard token rotated by a server restart" in dockerfile - - -def test_agent_image_runs_execution_safety_patch_and_regressions(): - dockerfiles = HERMES.parents[1] / "dockerfiles" - dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8") - dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text( - encoding="utf-8" - ) - - for name in ( - "patch-hermes-execution-safety.py", - "hermes-execution-safety-regression.py", - ): - assert f"COPY dockerfiles/{name}" in dockerfile - assert f"!dockerfiles/{name}" in dockerignore - assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in ( - dockerfile - ) - assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in ( - dockerfile - ) - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [{"protocol": "TCP", "port": 9010}], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) - - -def test_owner_agent_installs_the_pinned_operator_toolchain(): - script = (SCRIPTS / "install_agent_tools.sh").read_text() - for value in [ - "flux", - "helm", - "kustomize", - "jq", - "yq", - "gh", - "vault", - "sops", - "age", - "age-keygen", - "k9s", - "terraform", - "go", - "gofmt", - ]: - assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script - assert ( - "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" - in script - ) - assert script.count("sha256sum -c -") == 1 - - deployment = _agent_deployment() - installer = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "install-agent-tools" - ) - assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] - assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) - init_config = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - init_command = init_config["command"][2] - assert "# Hermes managed operator PATH." in init_command - assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command - assert 'chmod 0644 "${profile_file}"' in init_command - - -def test_owner_agent_uses_only_the_canonical_hostname(): - paths = [ - HERMES / "agent-configmap.yaml", - HERMES / "agent-deployment.yaml", - HERMES / "agent-ingress.yaml", - Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", - ] - for path in paths: - content = path.read_text() - assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content - - -def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - display = config["display"] - assert display["resume_exchanges"] >= 10000 - assert display["resume_max_user_chars"] >= 10000000 - assert display["resume_max_assistant_chars"] >= 10000000 - assert config["agent"]["max_turns"] == 180 - assert config["delegation"]["max_iterations"] == 120 - - -def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): - source = tmp_path / "auth.py" - destination = tmp_path / "patched/auth.py" - source.write_text( - 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', - encoding="utf-8", - ) - auth_patch.patch(source, destination) - content = destination.read_text() - assert 'os.environ.get("HERMES_AUTH_FILE"' in content - - -def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "auth.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - auth_patch.patch(source, tmp_path / "patched.py") - - -def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): - provider = tmp_path / "runtime_provider.py" - provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") - provider_out = tmp_path / "patched/runtime_provider.py" - codex_runtime_patch.patch_provider(provider, provider_out) - assert '"api_mode": "codex_app_server"' in provider_out.read_text() - - session = tmp_path / "codex_app_server_session.py" - session.write_text( - codex_runtime_patch.SESSION_SIGNATURE_BEFORE - + codex_runtime_patch.SESSION_REQUEST_BEFORE, - encoding="utf-8", - ) - session_out = tmp_path / "patched/codex_app_server_session.py" - codex_runtime_patch.patch_session(session, session_out) - session_content = session_out.read_text() - assert 'turn_params["model"] = model' in session_content - assert 'turn_params["effort"] = effort' in session_content - assert '"approvalPolicy": "never"' in session_content - assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content - - turn = tmp_path / "codex_runtime.py" - turn.write_text( - codex_runtime_patch.FALLBACK_CONTEXT_BEFORE - + codex_runtime_patch.TURN_BEFORE, - encoding="utf-8", - ) - turn_out = tmp_path / "patched/codex_runtime.py" - codex_runtime_patch.patch_turn(turn, turn_out) - turn_content = turn_out.read_text() - assert "model=str(getattr(agent" in turn_content - assert "build_cross_provider_codex_prompt" in turn_content - - fallback = tmp_path / "chat_completion_helpers.py" - fallback.write_text( - codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, - encoding="utf-8", - ) - fallback_out = tmp_path / "patched/chat_completion_helpers.py" - codex_runtime_patch.patch_fallback(fallback, fallback_out) - fallback_content = fallback_out.read_text() - assert 'agent.api_mode = "codex_app_server"' in fallback_content - assert "agent._codex_cross_provider_fallback = True" in fallback_content - - loop = tmp_path / "conversation_loop.py" - loop.write_text( - codex_runtime_patch.FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.STREAM_RECOVERY_BEFORE, - encoding="utf-8", - ) - loop_out = tmp_path / "patched/conversation_loop.py" - codex_runtime_patch.patch_loop(loop, loop_out) - loop_content = loop_out.read_text() - assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 - assert "build_cross_provider_codex_prompt" in loop_content - retry_dispatch = loop_content.index( - "Fallback activation happens inside this retry loop" - ) - api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) - assert api_kwargs == -1 or retry_dispatch < api_kwargs - assert "Provider stream ended before a complete response" in loop_content - assert "_is_transport_stub" in loop_content - assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content - - auxiliary = tmp_path / "auxiliary_client.py" - auxiliary.write_text( - codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - auxiliary_out = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) - auxiliary_content = auxiliary_out.read_text() - assert 'os.environ.get("CODEX_HOME"' in auxiliary_content - assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content - assert "never creates a metered API-key lane" in auxiliary_content - - -def test_agent_mounts_codex_auxiliary_runtime_patch(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-codex-runtime" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/auxiliary_client.py", - "/patched/auxiliary_client.py", - ] - expected_mount = { - "name": "codex-runtime-patch", - "mountPath": "/opt/hermes/agent/auxiliary_client.py", - "subPath": "auxiliary_client.py", - } - containers = {item["name"]: item for item in pod["containers"]} - for name in ("hermes", "terminal"): - assert expected_mount in containers[name]["volumeMounts"] - - -def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( - tmp_path: Path, - monkeypatch, -): - codex_home = tmp_path / ".codex" - codex_home.mkdir() - (codex_home / "auth.json").write_text( - json.dumps({"tokens": {"access_token": "cli-access-token"}}), - encoding="utf-8", - ) - monkeypatch.setenv("CODEX_HOME", str(codex_home)) - - source = tmp_path / "auxiliary_client.py" - source.write_text( - "import json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def read_token():\n" - " try:\n" - " raise RuntimeError('Hermes provider store intentionally empty')\n" - + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - destination = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(source, destination) - namespace: dict = {} - exec(compile(destination.read_text(), str(destination), "exec"), namespace) - - assert namespace["read_token"]() == "cli-access-token" - - -def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "runtime_provider.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") - - -def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): - config = tmp_path / "config.yaml" - config.write_text("model: {}\n", encoding="utf-8") - codex_home = tmp_path / ".codex" - calls = [] - - class Report: - errors = [] - - @staticmethod - def summary(): - return "configured" - - def migrate(value, **kwargs): - calls.append((value, kwargs)) - return Report() - - client_config.configure_codex_runtime(config, migrate, codex_home) - - assert calls[0][1]["default_permission_profile"] is None - assert calls[0][1]["codex_home"] == codex_home - content = (codex_home / "config.toml").read_text(encoding="utf-8") - assert 'approval_policy = "never"' in content - assert 'sandbox_mode = "danger-full-access"' in content - assert "default_permissions" not in content - - -def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): - config = tmp_path / "config.toml" - config.write_text( - 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', - encoding="utf-8", - ) - - client_config.configure_codex_owner_permissions(config) - client_config.configure_codex_owner_permissions(config) - - content = config.read_text(encoding="utf-8") - assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 - assert content.count('approval_policy = "never"') == 1 - assert content.count('sandbox_mode = "danger-full-access"') == 1 - assert "default_permissions" not in content - assert "[features]\nhooks = true" in content - - -def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): - source = tmp_path / "server.py" - destination = tmp_path / "patched/server.py" - source.write_text( - "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", - encoding="utf-8", - ) - - tui_gateway_patch.patch(source, destination) - - content = destination.read_text(encoding="utf-8") - assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content - assert 'configured = 180.0' in content - assert "return max(30.0, min(configured, 900.0))" in content - assert "timeout: float | None = None" in content - assert "ready.wait(timeout=wait_timeout)" in content - assert "def unchanged():" in content - - -def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "server.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - tui_gateway_patch.patch(source, tmp_path / "patched.py") - - -def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): - source = '' - content = ttyd_patch.patch_html(source) - assert 'id="atlas-ttyd-clipboard"' in content - assert "navigator.clipboard.writeText(text)" in content - assert "class AtlasRecoveringWebSocket" in content - assert "window.location.reload()" in content - assert "event.stopImmediatePropagation()" in content - - -def test_ttyd_patch_fails_closed_on_upstream_drift(): - with pytest.raises(RuntimeError, match="context changed"): - ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_cli_provenance.py b/testing/tests/test_hermes_cli_provenance.py new file mode 100644 index 00000000..506ffad1 --- /dev/null +++ b/testing/tests/test_hermes_cli_provenance.py @@ -0,0 +1,313 @@ +"""Run provenance and first-writer conflict behavior.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + _install_terminal_recovery_db, + _pending_terminal_record, + errno, + json, + lanes, + nullcontext, + os, + pytest, + stat, + sys, +) + + +def test_legacy_valid_retirement_staging_is_promoted_without_reexecution( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + staged = board / f".retire.{'d' * 32}.0" + record = _pending_terminal_record( + "cassandra", "t_legacy_staged", 56, "legacy staged result" + ) + lanes.atomic_json(staged, record) + task = SimpleNamespace( + id="t_legacy_staged", + status="running", + current_run_id=56, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + + assert lanes._has_pending_finalization( + "cassandra", "t_legacy_staged", 56 + ) is True + assert lanes.recover_pending_finalizations() == 1 + assert completions == [record["result"]] + assert reclaims == [] + assert not staged.exists() + assert lanes.recover_pending_finalizations() == 0 + assert completions == [record["result"]] + + +def test_done_result_bytes_do_not_authorize_a_different_run_journal( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, stale = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_provenance"), + board="cassandra", + task_id="t_provenance", + run_id=1, + structured=_completed_result("same bytes"), + summary="same bytes", + metadata={}, + ) + task = SimpleNamespace( + id="t_provenance", + status="done", + current_run_id=None, + completed_run_id=2, + result=stale["result"], + assignee="cli-auto", + ) + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=lambda *_args, **_kwargs: pytest.fail( + "a done task with different run provenance must not be completed" + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + assert not path.exists() + assert not list(path.parent.glob("*.terminal.committed.json")) + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + conflict = json.loads(conflicts[0].read_text()) + assert conflict["expected_run_id"] == 1 + assert conflict["result"] == task.result + + +@pytest.mark.parametrize( + "swap_point", + [ + "during-complete", + "after-db-before-promote", + "before-committed-create", + "before-pending-retire", + ], +) +def test_terminal_first_writer_preserves_valid_replacement_conflicts( + tmp_path: Path, + monkeypatch, + swap_point: str, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, old_record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_first_writer"), + board="cassandra", + task_id="t_first_writer", + run_id=27, + structured=_completed_result("first result"), + summary="first result", + metadata={"writer": "first"}, + ) + replacement_structured = _completed_result("replacement result") + replacement_record = { + "board": "cassandra", + "task_id": "t_first_writer", + "expected_run_id": 27, + "result": json.dumps(replacement_structured, sort_keys=True), + "summary": "replacement result", + "metadata": {"writer": "replacement"}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + replacement = path.with_name(f"replacement-{swap_point}.tmp") + lanes.atomic_json(replacement, replacement_record) + swapped = {"value": False} + + def swap_pending(): + if not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + + task = SimpleNamespace( + id="t_first_writer", + status="running", + result=None, + current_run_id=27, + assignee="cli-auto", + ) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + if swap_point == "during-complete": + swap_pending() + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + if swap_point == "after-db-before-promote": + real_promote = lanes._promote_prepared_evidence + + def swap_then_promote(*args, **kwargs): + swap_pending() + return real_promote(*args, **kwargs) + + monkeypatch.setattr(lanes, "_promote_prepared_evidence", swap_then_promote) + elif swap_point == "before-committed-create": + real_create = lanes._write_json_noreplace + + def swap_before_committed(path_arg, value): + if path_arg.name.endswith(".terminal.committed.json"): + swap_pending() + return real_create(path_arg, value) + + monkeypatch.setattr(lanes, "_write_json_noreplace", swap_before_committed) + elif swap_point == "before-pending-retire": + real_retire = lanes._retire_snapshot + + def swap_before_retire(path_arg, snapshot): + if path_arg == path: + swap_pending() + return real_retire(path_arg, snapshot) + + monkeypatch.setattr(lanes, "_retire_snapshot", swap_before_retire) + + assert lanes._finalize_terminal_record(fake_db, path, old_record) == "committed" + + assert swapped["value"] is True + assert task.result == old_record["result"] + assert not path.exists() + committed = list(path.parent.glob("*.terminal.committed.json")) + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + prepared = list(path.parent.glob("*.terminal.prepared-*.json")) + assert len(committed) == 1 + assert len(conflicts) == 1 + assert prepared == [] + assert committed[0].stat().st_mode & 0o777 == 0o600 + assert conflicts[0].stat().st_mode & 0o777 == 0o600 + committed_document = json.loads(committed[0].read_text(encoding="utf-8")) + conflict_document = json.loads(conflicts[0].read_text(encoding="utf-8")) + assert committed_document["result"] == old_record["result"] + assert committed_document["kanban_state"] == "committed" + assert conflict_document["result"] == replacement_record["result"] + assert conflict_document["metadata"] == {"writer": "replacement"} + assert conflict_document["kanban_state"] == "conflict" + + +def test_terminal_commit_directory_fsync_failure_recovers_from_prepared_evidence( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_post_db_fsync"), + board="cassandra", + task_id="t_post_db_fsync", + run_id=28, + structured=_completed_result("durable DB winner"), + summary="durable DB winner", + metadata={}, + ) + task = SimpleNamespace( + id="t_post_db_fsync", + status="running", + result=None, + current_run_id=28, + completed_run_id=None, + assignee="cli-auto", + ) + replacement_structured = _completed_result("replacement after DB commit") + replacement_record = { + "board": "cassandra", + "task_id": "t_post_db_fsync", + "expected_run_id": 28, + "result": json.dumps(replacement_structured, sort_keys=True), + "summary": "replacement after DB commit", + "metadata": {"writer": "replacement"}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + replacement = path.with_name("post-db-fsync-replacement.tmp") + lanes.atomic_json(replacement, replacement_record) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.completed_run_id = kwargs["expected_run_id"] + task.current_run_id = None + os.replace(replacement, path) + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + real_fsync = lanes.os.fsync + failed = {"value": False} + + def fail_committed_directory_fsync(descriptor): + if ( + stat.S_ISDIR(os.fstat(descriptor).st_mode) + and list(path.parent.glob("*.terminal.committed.json")) + and not failed["value"] + ): + failed["value"] = True + raise OSError(errno.ENOSPC, "post-DB committed directory fsync failed") + real_fsync(descriptor) + + monkeypatch.setattr(lanes.os, "fsync", fail_committed_directory_fsync) + + with pytest.raises(OSError, match="post-DB committed"): + lanes._finalize_terminal_record(fake_db, path, record) + + assert task.status == "done" + assert task.result == record["result"] + assert path.exists() + assert json.loads(path.read_text(encoding="utf-8"))["result"] == ( + replacement_record["result"] + ) + assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1 + + monkeypatch.setattr(lanes.os, "fsync", real_fsync) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + assert lanes.recover_pending_finalizations() == 1 + assert not path.exists() + assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] + committed = list(path.parent.glob("*.terminal.committed.json")) + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + assert len(committed) == 1 + assert len(conflicts) == 1 + assert json.loads(committed[0].read_text())["result"] == record["result"] + assert json.loads(conflicts[0].read_text())["result"] == replacement_record["result"] diff --git a/testing/tests/test_hermes_cli_provider_sessions.py b/testing/tests/test_hermes_cli_provider_sessions.py new file mode 100644 index 00000000..bb034f74 --- /dev/null +++ b/testing/tests/test_hermes_cli_provider_sessions.py @@ -0,0 +1,301 @@ +"""Codex, Claude, and worker-process session behavior.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + json, + lanes, + os, + signal, + sys, +) + + +def test_claude_session_is_reserved_before_first_process(tmp_path: Path, monkeypatch): + state: dict = {} + state_file = tmp_path / "state.json" + observed = {} + + def fake_stream(command, **kwargs): + observed["command"] = command + observed["state"] = json.loads(state_file.read_text()) + return lanes.ProcessResult(0, "", {"status": "completed"}, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) + + lanes.run_provider( + route, + "Do the work.", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + + reserved = observed["state"]["claude_session_id"] + assert reserved + assert ["--session-id", reserved] == observed["command"][-4:-2] + + +def test_claude_recovery_uses_verified_error_state_machine(tmp_path: Path, monkeypatch): + state = {"claude_session_id": "d4f91f59-88cb-43d0-a64f-f3284688fe9e", "claude_started": True} + state_file = tmp_path / "state.json" + calls = [] + + def fake_stream(command, **kwargs): + calls.append(command) + if len(calls) == 1: + return lanes.ProcessResult( + 1, + "No conversation found with session ID: d4f91f59-88cb-43d0-a64f-f3284688fe9e", + None, + False, + ) + return lanes.ProcessResult(0, "", {"status": "completed"}, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) + lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60) + + assert "--resume" in calls[0] + assert "--session-id" in calls[1] + assert calls[0][calls[0].index("--resume") + 1] == calls[1][calls[1].index("--session-id") + 1] + + +def test_claude_collision_never_double_starts_a_reserved_session(tmp_path: Path, monkeypatch): + state = {"claude_session_id": "24927cee-f3da-4bcf-b624-602c22860155"} + state_file = tmp_path / "state.json" + calls = [] + + def fake_stream(command, **kwargs): + calls.append(command) + if len(calls) == 1: + return lanes.ProcessResult(1, "Session ID already in use", None, False) + return lanes.ProcessResult(0, "", {"status": "completed"}, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ()) + lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60) + + assert "--session-id" in calls[0] + assert "--resume" in calls[1] + assert calls[0][calls[0].index("--session-id") + 1] == calls[1][calls[1].index("--resume") + 1] + + +def test_codex_thread_started_is_persisted_before_completion(tmp_path: Path): + state: dict = {} + state_file = tmp_path / "state.json" + + lanes._event_payload( + "codex", + json.dumps({"type": "thread.started", "thread_id": "thread-123"}), + state, + state_file, + ) + + assert json.loads(state_file.read_text())["codex_thread_id"] == "thread-123" + + +def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch): + state = {"codex_thread_id": "failed-thread"} + state_file = tmp_path / "state.json" + calls = [] + + def fake_stream(command, **kwargs): + calls.append(command) + if len(calls) == 1: + return lanes.ProcessResult( + 1, + "thread/resume failed: no rollout found for thread id failed-thread", + None, + False, + ) + return lanes.ProcessResult(0, "", {"status": "completed"}, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ) + + result = lanes.run_provider( + route, + "Continue.", + tmp_path, + state, + state_file, + tmp_path / "log", + lambda _: True, + 60, + ) + + assert result.returncode == 0 + assert "resume" in calls[0] + assert "resume" not in calls[1] + assert "codex_thread_id" not in json.loads(state_file.read_text()) + + +def test_codex_result_files_are_unique_and_prior_turns_are_retained( + tmp_path: Path, + monkeypatch, +): + state = {"run_id": 17} + state_file = tmp_path / "task.json" + result_paths = [] + + def fake_stream(command, **_kwargs): + result_path = Path(command[command.index("-o") + 1]) + result_path.write_text( + json.dumps( + { + "status": "completed", + "summary": f"turn {len(result_paths) + 1}", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + } + ), + encoding="utf-8", + ) + result_paths.append(result_path) + return lanes.ProcessResult(0, "", None, False) + + monkeypatch.setattr(lanes, "stream_process", fake_stream) + route = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ) + + first = lanes.run_provider( + route, + "First turn.", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + second = lanes.run_provider( + route, + "Second turn.", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + + assert first.structured["summary"] == "turn 1" + assert second.structured["summary"] == "turn 2" + assert len(set(result_paths)) == 2 + assert all(path.exists() for path in result_paths) + assert all(path.stat().st_mode & 0o777 == 0o600 for path in result_paths) + + +def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path): + result = lanes.stream_process( + [sys.executable, "-c", "print('authentication work completed')"], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "worker.log", + state={}, + state_file=tmp_path / "state.json", + heartbeat=lambda _note: True, + max_runtime=60, + ) + + assert result.returncode == 0 + assert result.capacity_failure is False + + +def test_worker_process_uses_isolated_process_session(monkeypatch, tmp_path: Path): + original_popen = lanes.subprocess.Popen + calls = [] + + def recording_popen(*args, **kwargs): + calls.append(kwargs) + return original_popen(*args, **kwargs) + + monkeypatch.setattr(lanes.subprocess, "Popen", recording_popen) + result = lanes.stream_process( + [sys.executable, "-c", "print('done')"], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "worker.log", + state={}, + state_file=tmp_path / "state.json", + heartbeat=lambda _note: True, + max_runtime=60, + ) + + assert result.returncode == 0 + assert calls[0]["start_new_session"] is True + + +def test_worker_process_group_is_killed_after_leader_exits(monkeypatch): + class FinishedProcess: + pid = 4321 + + @staticmethod + def poll(): + return 0 + + signals = [] + monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig))) + + lanes._terminate_worker_process(FinishedProcess()) + + assert signals == [(4321, signal.SIGKILL)] + + +def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): + class RunningProcess: + pid = 4321 + running = True + + @classmethod + def poll(cls): + return None if cls.running else 0 + + @classmethod + def wait(cls, timeout): + assert timeout == 10 + cls.running = False + return 0 + + group_signals = [] + process_signals = [] + monkeypatch.setattr( + lanes, + "_descendant_processes", + lambda _pid: {5000: (5000, 12345)}, + ) + monkeypatch.setattr(lanes, "_process_identity_matches", lambda _pid, _start: True) + monkeypatch.setattr( + lanes.os, + "killpg", + lambda process_group, sig: group_signals.append((process_group, sig)), + ) + monkeypatch.setattr( + lanes.os, + "kill", + lambda pid, sig: process_signals.append((pid, sig)), + ) + + lanes._terminate_worker_process(RunningProcess()) + + assert set(group_signals) == { + (4321, signal.SIGTERM), + (5000, signal.SIGTERM), + (4321, signal.SIGKILL), + (5000, signal.SIGKILL), + } + assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] diff --git a/testing/tests/test_hermes_cli_quarantine.py b/testing/tests/test_hermes_cli_quarantine.py new file mode 100644 index 00000000..f8a3656e --- /dev/null +++ b/testing/tests/test_hermes_cli_quarantine.py @@ -0,0 +1,375 @@ +"""Identity-safe quarantine and replacement races.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + _install_terminal_recovery_db, + _pending_terminal_record, + hashlib, + json, + lanes, + nullcontext, + os, + pytest, + stat, + sys, +) + + +@pytest.mark.parametrize( + "structured", + [ + {"status": "completed", "blockers": {}}, + {**_completed_result(), "changed_files": "src/a.py"}, + {**_completed_result(), "unexpected": True}, + {**_completed_result(), "blockers": ["work remains"]}, + {**_completed_result("tests are still running")}, + ], +) +def test_terminal_record_requires_exact_completed_result_contract(structured): + record = { + "board": "cassandra", + "task_id": "t_schema", + "expected_run_id": 1, + "result": json.dumps(structured), + "summary": str(structured.get("summary") or "done"), + "metadata": {}, + "kanban_state": "pending", + } + + assert lanes._terminal_record_valid(record) is False + + +def test_minimal_completed_result_with_mapping_blockers_quarantines_without_completion( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_schema"), 6) + lanes.atomic_json( + path, + { + "board": "cassandra", + "task_id": "t_schema", + "expected_run_id": 6, + "result": json.dumps({"status": "completed", "blockers": {}}), + "summary": "done", + "metadata": {}, + "kanban_state": "pending", + }, + ) + task = SimpleNamespace( + id="t_schema", status="running", current_run_id=6, assignee="cli-auto" + ) + completed = [] + reclaimed = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=lambda *_args, **_kwargs: completed.append(True), + reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 0 + assert completed == [] + assert reclaimed == [True] + assert not path.exists() + quarantined = list((path.parent / "quarantine").glob("*.quarantine")) + assert len(quarantined) == 1 + assert quarantined[0].stat().st_mode & 0o777 == 0o600 + + +def test_quarantine_avoids_symlink_and_mode_collision_destinations( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_collision"), 2) + path.parent.mkdir(parents=True) + path.write_bytes(b"invalid") + quarantine = path.parent / "quarantine" + quarantine.mkdir() + fingerprint = hashlib.sha256(b"invalid").hexdigest()[:16] + base = f"{path.name}.malformed-payload.{fingerprint}" + victim = tmp_path / "victim" + victim.write_text("unchanged", encoding="utf-8") + (quarantine / f"{base}.0.quarantine").symlink_to(victim) + collision = quarantine / f"{base}.1.quarantine" + collision.write_text("attacker collision", encoding="utf-8") + collision.chmod(0o644) + monkeypatch.setattr( + lanes.os, + "chmod", + lambda *_args, **_kwargs: pytest.fail("quarantine must not chmod foreign inodes"), + ) + + destination = lanes._quarantine_terminal( + path, + lanes._terminal_identity(path), + "malformed-payload", + ) + + assert not path.exists() + assert destination.name == f"{base}.2.quarantine" + assert destination.stat().st_mode & 0o777 == 0o600 + diagnostic = json.loads(destination.read_text(encoding="utf-8")) + assert diagnostic["size"] == len(b"invalid") + assert diagnostic["sha256"] == hashlib.sha256(b"invalid").hexdigest() + assert diagnostic["source_kind"] == "regular" + assert victim.read_text(encoding="utf-8") == "unchanged" + assert collision.read_text(encoding="utf-8") == "attacker collision" + assert collision.stat().st_mode & 0o777 == 0o644 + + +def test_hardlinked_terminal_source_is_never_chmodded_or_copied_as_data( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_hardlink"), 5) + path.parent.mkdir(parents=True) + foreign = tmp_path / "foreign" + foreign.write_text("foreign inode contents", encoding="utf-8") + foreign.chmod(0o644) + os.link(foreign, path) + monkeypatch.setattr( + lanes.os, + "chmod", + lambda *_args, **_kwargs: pytest.fail("hardlinked source must not be chmodded"), + ) + + destination = lanes._quarantine_terminal( + path, + lanes._terminal_identity(path), + "malformed-payload", + ) + + assert not path.exists() + assert foreign.read_text(encoding="utf-8") == "foreign inode contents" + assert foreign.stat().st_mode & 0o777 == 0o644 + assert destination.stat().st_mode & 0o777 == 0o600 + diagnostic = json.loads(destination.read_text(encoding="utf-8")) + assert diagnostic["source_kind"] == "hardlink" + assert "foreign inode contents" not in destination.read_text(encoding="utf-8") + + +def test_quarantine_preserves_and_replays_an_atomic_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_swap"), 21) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed") + identity = lanes._terminal_identity(path) + assert identity is not None + structured = _completed_result("replacement result") + replacement_record = { + "board": "cassandra", + "task_id": "t_swap", + "expected_run_id": 21, + "result": json.dumps(structured, sort_keys=True), + "summary": structured["summary"], + "metadata": {}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + replacement = path.with_name("replacement.tmp") + replacement.write_text( + json.dumps(replacement_record, sort_keys=True), + encoding="utf-8", + ) + replacement.chmod(0o600) + real_fsync = lanes.os.fsync + swapped = {"value": False} + + def swap_on_quarantine_directory_fsync(descriptor): + descriptor_stat = os.fstat(descriptor) + if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + real_fsync(descriptor) + + monkeypatch.setattr(lanes.os, "fsync", swap_on_quarantine_directory_fsync) + + lanes._quarantine_terminal(path, identity, "malformed-payload") + + assert swapped["value"] is True + assert path.exists() + assert lanes._terminal_record_valid( + lanes._load_terminal_json(path, identity), identity + ) + + task = SimpleNamespace( + id="t_swap", status="running", current_run_id=21, result=None, assignee="cli-auto" + ) + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert task.status == "done" + assert not path.exists() + + +def test_recovery_quarantines_the_loaded_inode_not_a_before_open_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_gap"), 51) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed-original") + replacement = path.with_name("replacement.valid") + lanes.atomic_json( + replacement, + _pending_terminal_record("cassandra", "t_gap", 51, "preserved replacement"), + ) + task = SimpleNamespace( + id="t_gap", + status="running", + current_run_id=51, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + real_quarantine = lanes._quarantine_terminal + swapped = {"value": False} + + def swap_before_quarantine_reopens(*args, **kwargs): + assert kwargs["snapshot"].prefix == b"malformed-original" + if not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + return real_quarantine(*args, **kwargs) + + monkeypatch.setattr(lanes, "_quarantine_terminal", swap_before_quarantine_reopens) + + assert lanes.recover_pending_finalizations() == 1 + assert swapped["value"] is True + assert task.status == "done" + assert len(completions) == 1 + assert reclaims == [] + assert not path.exists() + committed = next(path.parent.glob("*.terminal.committed.json")) + assert json.loads(committed.read_text())["result"] == completions[0] + + +def test_recovery_replays_replacement_installed_after_quarantine_pins_source( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_after_open"), 52) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed-original") + replacement = path.with_name("replacement.valid") + lanes.atomic_json( + replacement, + _pending_terminal_record( + "cassandra", "t_after_open", 52, "replacement after open" + ), + ) + task = SimpleNamespace( + id="t_after_open", + status="running", + current_run_id=52, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + real_fsync = lanes.os.fsync + swapped = {"value": False} + + def swap_after_quarantine_open(descriptor): + descriptor_stat = os.fstat(descriptor) + if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + real_fsync(descriptor) + + monkeypatch.setattr(lanes.os, "fsync", swap_after_quarantine_open) + + assert lanes.recover_pending_finalizations() == 1 + assert swapped["value"] is True + assert task.status == "done" + assert len(completions) == 1 + assert reclaims == [] + assert not path.exists() + + +def test_recovery_survives_repeated_invalid_then_valid_replacements( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_churn"), 53) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed-zero") + second = path.with_name("replacement.invalid") + second.write_bytes(b"malformed-one") + valid = path.with_name("replacement.valid") + lanes.atomic_json( + valid, + _pending_terminal_record("cassandra", "t_churn", 53, "valid after churn"), + ) + task = SimpleNamespace( + id="t_churn", + status="running", + current_run_id=53, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + real_quarantine = lanes._quarantine_terminal + replacements = [second, valid] + pinned = [] + + def churn_before_quarantine(*args, **kwargs): + pinned.append(kwargs["snapshot"].prefix) + if replacements: + os.replace(replacements.pop(0), path) + return real_quarantine(*args, **kwargs) + + monkeypatch.setattr(lanes, "_quarantine_terminal", churn_before_quarantine) + + assert lanes.recover_pending_finalizations() == 1 + assert pinned == [b"malformed-zero", b"malformed-one"] + assert replacements == [] + assert task.status == "done" + assert len(completions) == 1 + assert reclaims == [] + assert not path.exists() + diagnostics = list((path.parent / "quarantine").glob("*.quarantine")) + assert len(diagnostics) == 2 diff --git a/testing/tests/test_hermes_cli_retention.py b/testing/tests/test_hermes_cli_retention.py new file mode 100644 index 00000000..1cf1c01b --- /dev/null +++ b/testing/tests/test_hermes_cli_retention.py @@ -0,0 +1,325 @@ +"""Bounded reads, artifact retention, and recovery cadence.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + json, + lanes, + nullcontext, + os, + pytest, + sys, +) + + +def test_invalid_utf8_journal_quarantines_and_does_not_stop_later_replay( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + bad = lanes._terminal_path(lanes.state_path("cassandra", "a_bad_utf8"), 31) + bad.parent.mkdir(parents=True) + bad.write_bytes(b"\xff\xfe\x80not-json") + good, _record = lanes._write_terminal_record( + lanes.state_path("cassandra", "z_good"), + board="cassandra", + task_id="z_good", + run_id=32, + structured=_completed_result("valid after invalid UTF-8"), + summary="valid after invalid UTF-8", + metadata={}, + ) + tasks = { + "a_bad_utf8": SimpleNamespace( + id="a_bad_utf8", + status="running", + current_run_id=31, + result=None, + assignee="cli-auto", + ), + "z_good": SimpleNamespace( + id="z_good", + status="running", + current_run_id=32, + result=None, + assignee="cli-auto", + ), + } + + class Connection: + def close(self): + return None + + def reclaim_task(_conn, task_id, **_kwargs): + tasks[task_id].status = "ready" + tasks[task_id].current_run_id = None + return True + + def complete_task(_conn, task_id, **kwargs): + tasks[task_id].status = "done" + tasks[task_id].result = kwargs["result"] + tasks[task_id].current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, task_id: tasks[task_id], + reclaim_task=reclaim_task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert not bad.exists() + assert not good.exists() + assert tasks["a_bad_utf8"].status == "ready" + assert tasks["z_good"].status == "done" + diagnostic_path = next((bad.parent / "quarantine").glob("*.quarantine")) + diagnostic = json.loads(diagnostic_path.read_text(encoding="utf-8")) + assert diagnostic["size"] == len(b"\xff\xfe\x80not-json") + assert diagnostic_path.stat().st_size < 4096 + + +@pytest.mark.parametrize( + "size", + [lanes.MAX_TERMINAL_RECORD_BYTES + 1, 16 * 1024 * 1024], +) +def test_oversized_sparse_journal_has_bounded_read_and_quarantine( + tmp_path: Path, + monkeypatch, + size: int, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + huge = lanes._terminal_path(lanes.state_path("cassandra", "a_huge"), 41) + huge.parent.mkdir(parents=True) + with huge.open("wb") as stream: + stream.seek(size - 1) + stream.write(b"\0") + good, _record = lanes._write_terminal_record( + lanes.state_path("cassandra", "z_after_huge"), + board="cassandra", + task_id="z_after_huge", + run_id=42, + structured=_completed_result("valid after oversized journal"), + summary="valid after oversized journal", + metadata={}, + ) + tasks = { + "a_huge": SimpleNamespace( + id="a_huge", + status="running", + current_run_id=41, + result=None, + assignee="cli-auto", + ), + "z_after_huge": SimpleNamespace( + id="z_after_huge", + status="running", + current_run_id=42, + result=None, + assignee="cli-auto", + ), + } + reads = [] + real_read_bounded = lanes._read_bounded + + def observe_read_limit(descriptor, limit): + reads.append((os.fstat(descriptor).st_size, limit)) + return real_read_bounded(descriptor, limit) + + monkeypatch.setattr(lanes, "_read_bounded", observe_read_limit) + + class Connection: + def close(self): + return None + + def reclaim_task(_conn, task_id, **_kwargs): + tasks[task_id].status = "ready" + tasks[task_id].current_run_id = None + return True + + def complete_task(_conn, task_id, **kwargs): + tasks[task_id].status = "done" + tasks[task_id].result = kwargs["result"] + tasks[task_id].current_run_id = None + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, task_id: tasks[task_id], + reclaim_task=reclaim_task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert not huge.exists() + assert not good.exists() + assert tasks["a_huge"].status == "ready" + assert tasks["z_after_huge"].status == "done" + diagnostics = list((huge.parent / "quarantine").glob("*.quarantine")) + assert len(diagnostics) == 1 + assert diagnostics[0].stat().st_size < 4096 + diagnostic = json.loads(diagnostics[0].read_text(encoding="utf-8")) + assert diagnostic["size"] == size + assert diagnostic["hashed_bytes"] == lanes.QUARANTINE_HASH_BYTES + assert diagnostic["hash_complete"] is False + assert (size, lanes.QUARANTINE_HASH_BYTES) in reads + assert (size, lanes.MAX_TERMINAL_RECORD_BYTES + 1) not in reads + assert max(limit for _file_size, limit in reads) <= ( + lanes.MAX_TERMINAL_RECORD_BYTES + 1 + ) + + +def test_recovery_does_not_parse_committed_journals_every_tick( + tmp_path: Path, + monkeypatch, +): + state_root = tmp_path / "cli-lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + committed = lanes._terminal_path( + lanes.state_path("cassandra", "t_done"), 1, "committed" + ) + lanes.atomic_json(committed, {"kanban_state": "committed"}) + reads = [] + monkeypatch.setattr(lanes, "load_json", lambda path: (reads.append(path) or {})) + monkeypatch.setitem( + sys.modules, + "hermes_cli", + SimpleNamespace(kanban_db=SimpleNamespace()), + ) + + assert lanes.recover_pending_finalizations() == 0 + assert reads == [] + + +def test_atomic_json_fsyncs_file_and_directory_and_uses_private_mode( + tmp_path: Path, + monkeypatch, +): + calls = [] + real_fsync = lanes.os.fsync + monkeypatch.setattr( + lanes.os, + "fsync", + lambda descriptor: (calls.append(descriptor), real_fsync(descriptor))[1], + ) + path = tmp_path / "state.json" + + lanes.atomic_json(path, {"safe": True}) + + assert json.loads(path.read_text()) == {"safe": True} + assert path.stat().st_mode & 0o777 == 0o600 + assert len(calls) >= 2 + + +def test_atomic_json_refuses_a_precreated_temp_symlink( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "state.json" + victim = tmp_path / "credential" + victim.write_text("do not overwrite", encoding="utf-8") + temporary = tmp_path / ".state.json.fixed.tmp" + temporary.symlink_to(victim) + monkeypatch.setattr(lanes.uuid, "uuid4", lambda: SimpleNamespace(hex="fixed")) + + with pytest.raises(FileExistsError): + lanes.atomic_json(path, {"unsafe": True}) + + assert victim.read_text(encoding="utf-8") == "do not overwrite" + assert not path.exists() + + +def test_artifact_gc_prunes_by_age_without_touching_pending_journals( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + quarantine = board / "quarantine" + quarantine.mkdir() + old_artifacts = [ + board / "t.run-1.candidate-1.json", + board / "t.run-1.provider-1.result.json", + board / "t.run-1.terminal.committed.json", + board / f"t.run-1.terminal.conflict-{'a' * 32}.json", + quarantine / "t.invalid.1234.quarantine", + ] + for artifact in old_artifacts: + artifact.write_text("{}", encoding="utf-8") + quarantine_symlink = quarantine / "attacker.quarantine" + quarantine_symlink.symlink_to(tmp_path / "missing-target") + pending = board / "t.run-1.terminal.pending.json" + pending.write_text("{}", encoding="utf-8") + prepared = board / f"t.run-1.terminal.prepared-{'b' * 32}.json" + prepared.write_text("{}", encoding="utf-8") + old_time = 100.0 + for artifact in old_artifacts: + os.utime(artifact, (old_time, old_time)) + os.utime(pending, (old_time, old_time)) + os.utime(prepared, (old_time, old_time)) + + assert lanes.gc_lane_artifacts( + now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 + ) == len(old_artifacts) + 1 + assert not any(artifact.exists() for artifact in old_artifacts) + assert not quarantine_symlink.is_symlink() + assert pending.exists() + assert prepared.exists() + + +def test_artifact_gc_prunes_oldest_by_count_and_total_bytes( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + files = [] + for sequence in range(3): + path = board / f"t.run-1.provider-{sequence}.result.json" + path.write_text("x" * 40, encoding="utf-8") + os.utime(path, (100 + sequence, 100 + sequence)) + files.append(path) + + assert lanes.gc_lane_artifacts( + now=200.0, max_age_seconds=1000, max_count=2, max_bytes=45 + ) == 2 + assert [path.exists() for path in files] == [False, False, True] + + +def test_artifact_gc_preserves_a_concurrent_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + path = board / "t.run-1.provider-1.result.json" + path.write_text("old", encoding="utf-8") + os.utime(path, (100, 100)) + replacement = board / "replacement.tmp" + replacement.write_text("new", encoding="utf-8") + real_unlink = lanes._unlink_artifact_if_same + swapped = {"value": False} + + def swap_before_identity_checked(candidate, observed): + if not swapped["value"]: + os.replace(replacement, candidate) + swapped["value"] = True + return real_unlink(candidate, observed) + + monkeypatch.setattr(lanes, "_unlink_artifact_if_same", swap_before_identity_checked) + + assert lanes.gc_lane_artifacts( + now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000 + ) == 0 + assert swapped["value"] is True + assert path.read_text(encoding="utf-8") == "new" diff --git a/testing/tests/test_hermes_cli_routing.py b/testing/tests/test_hermes_cli_routing.py new file mode 100644 index 00000000..67782339 --- /dev/null +++ b/testing/tests/test_hermes_cli_routing.py @@ -0,0 +1,155 @@ +"""Switchyard routing and provider-health behavior.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + _SwitchyardResponse, + json, + lanes, +) + + +def test_auto_lane_uses_switchyard_worker_decision(): + observed = {} + + def route(request, timeout): + observed["payload"] = json.loads(request.data) + observed["timeout"] = timeout + return _SwitchyardResponse("worker/claude/claude-opus-5/xhigh") + + route = lanes.select_route( + "Perform the security review.", + "cli-auto", + open_request=route, + ) + + assert route.provider == "claude" + assert route.model == "claude-opus-5" + assert route.effort == "xhigh" + assert route.classifier == "switchyard-classifier" + assert observed["payload"]["model"] == "atlas/worker/auto/maximum" + assert observed["timeout"] == 60 + + +def test_manual_lane_is_still_enforced_by_switchyard(): + observed = {} + + def route(request, timeout): + observed["payload"] = json.loads(request.data) + return _SwitchyardResponse( + "worker/claude/claude-sonnet-5/high", "manual route" + ) + + route = lanes.select_route( + "Implement it.", + "cli-claude-high", + open_request=route, + ) + + assert route.provider == "claude" + assert route.effort == "high" + assert route.classifier == "switchyard-manual" + assert route.reason == "manual route" + assert observed["payload"]["model"] == "atlas/worker/manual/claude/high" + + +def test_cross_provider_retry_passes_failed_provider_to_switchyard(): + observed = {} + + def route(request, timeout): + observed["payload"] = json.loads(request.data) + return _SwitchyardResponse("worker/claude/claude-sonnet-5/high") + + route = lanes.select_route( + "Retry after capacity exhaustion.", + "cli-auto", + exclude_provider="codex", + open_request=route, + ) + + assert route.provider == "claude" + assert route.classifier == "switchyard-classifier" + content = observed["payload"]["messages"][0]["content"] + assert "codex provider failed or exhausted capacity" in content + + +def test_classifier_cannot_select_a_freshly_excluded_provider(): + payloads = [] + + def route(request, timeout): + payload = json.loads(request.data) + payloads.append(payload) + if len(payloads) == 1: + return _SwitchyardResponse("worker/claude/opus/xhigh") + return _SwitchyardResponse("worker/codex/sol/xhigh", "healthy route") + + selected = lanes.select_route( + "Perform a consequential review.", + "cli-auto", + exclude_provider="claude", + exclude_reason="is unavailable according to fresh native health", + open_request=route, + ) + + assert selected.provider == "codex" + assert selected.effort == "xhigh" + assert selected.classifier == "switchyard-classifier-health-guard" + assert payloads[0]["model"] == "atlas/worker/auto/maximum" + assert payloads[1]["model"] == "atlas/worker/manual/codex/xhigh" + assert "claude provider is unavailable" in payloads[0]["messages"][0]["content"] + + +def test_fresh_native_health_excludes_only_one_proven_down_provider( + tmp_path: Path, monkeypatch +): + paths = { + "codex": tmp_path / "codex.json", + "claude": tmp_path / "claude.json", + } + paths["codex"].write_text('{"state":"available"}\n', encoding="utf-8") + paths["claude"].write_text('{"state":"unavailable"}\n', encoding="utf-8") + monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) + + now = max(path.stat().st_mtime for path in paths.values()) + assert lanes.fresh_unavailable_provider(now=now) == "claude" + + paths["codex"].write_text('{"state":"unavailable"}\n', encoding="utf-8") + now = max(path.stat().st_mtime for path in paths.values()) + assert lanes.fresh_unavailable_provider(now=now) is None + + +def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch): + paths = { + "codex": tmp_path / "codex.json", + "claude": tmp_path / "claude.json", + } + paths["codex"].write_text( + '{"state":"available","authenticated":true}\n', encoding="utf-8" + ) + paths["claude"].write_text( + '{"state":"unavailable","authenticated":false}\n', encoding="utf-8" + ) + monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) + + stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60 + assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude" + + paths["claude"].write_text( + '{"state":"unavailable","authenticated":true}\n', encoding="utf-8" + ) + transient_stale = paths["claude"].stat().st_mtime + 10 * 60 + assert lanes.fresh_unavailable_provider(now=transient_stale) is None + + +def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch): + """Kanban workers must not fall back to credential-free persistent homes.""" + monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "/runtime-access/claude") + + env = lanes._base_env() + + assert env["HOME"] == str(lanes.DATA_ROOT / "home") + assert env["CODEX_HOME"] == "/runtime-access/codex" + assert env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert env["GIT_TERMINAL_PROMPT"] == "0" diff --git a/testing/tests/test_hermes_cli_run_identity.py b/testing/tests/test_hermes_cli_run_identity.py new file mode 100644 index 00000000..35aca4f2 --- /dev/null +++ b/testing/tests/test_hermes_cli_run_identity.py @@ -0,0 +1,217 @@ +"""Canonical SQLite run-identity contracts for every terminal artifact class.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + _completed_result, + _pending_terminal_record, + lanes, + pytest, + sys, +) + + +class _NoDatabase: + """Fail if malformed run authority reaches a Kanban transaction.""" + + def __init__(self) -> None: + self.accesses = [] + + def scoped_current_board(self, *_args, **_kwargs): + self.accesses.append("scope") + raise AssertionError("noncanonical authority reached the database") + + +@pytest.mark.parametrize( + "value", + [ + False, + True, + 0, + -1, + 2**63, + "0", + "01", + "9223372036854775808", + "9" * 150, + "+1", + " 1", + ], +) +def test_run_identity_rejects_noncanonical_or_non_sqlite_values(value): + assert lanes.canonical_run_id(value) is None + + +@pytest.mark.parametrize("value", [1, 2**63 - 1, "1", "9223372036854775807"]) +def test_run_identity_accepts_the_positive_sqlite_int64_domain(value): + assert lanes.canonical_run_id(value) == int(value) + + +@pytest.mark.parametrize("run_id", [False, True, 0, -1, 2**63, 10**149]) +def test_terminal_writer_refuses_invalid_runtime_run_ids(tmp_path: Path, run_id): + with pytest.raises(ValueError, match="SQLite int64"): + lanes._write_terminal_record( + lanes.state_path("cassandra", "t_invalid_writer"), + board="cassandra", + task_id="t_invalid_writer", + run_id=run_id, + structured=_completed_result(), + summary="completed", + metadata={}, + ) + + +@pytest.mark.parametrize( + ("state", "suffix", "parser"), + [ + ("pending", "terminal.pending.json", "_terminal_identity"), + ("committed", "terminal.committed.json", "_terminal_identity"), + ( + "prepared", + f"terminal.prepared-{'a' * 32}.json", + "_terminal_evidence_identity", + ), + ( + "conflict", + f"terminal.conflict-{'a' * 32}.json", + "_terminal_evidence_identity", + ), + ], +) +@pytest.mark.parametrize("encoded", ["0", "01", "9223372036854775808", "9" * 150]) +def test_every_terminal_filename_class_rejects_noncanonical_run_ids( + tmp_path: Path, + monkeypatch, + state: str, + suffix: str, + parser: str, + encoded: str, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + path = board / f"t_filename.run-{encoded}.{suffix}" + + assert getattr(lanes, parser)(path) is None, state + + +@pytest.mark.parametrize("run_id", [False, True, 0, -1, 2**63, 10**149]) +@pytest.mark.parametrize("state", ["pending", "prepared", "committed", "conflict"]) +def test_every_terminal_record_class_rejects_invalid_run_ids(run_id, state): + record = _pending_terminal_record( + "cassandra", "t_record", run_id, "invalid identity" + ) + record["kanban_state"] = state + + assert lanes._terminal_record_valid(record) is False + + +@pytest.mark.parametrize("run_id", [False, 0, 2**63, 10**149]) +def test_retirement_staging_rejects_invalid_document_run_ids( + tmp_path: Path, + monkeypatch, + run_id, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + staged = board / f".retire.{'a' * 16}.{'b' * 16}.0" + lanes.atomic_json( + staged, + _pending_terminal_record( + "cassandra", "t_staged", run_id, "invalid staged identity" + ), + ) + snapshot = lanes._open_terminal_recovery_snapshot(staged) + assert snapshot is not None + try: + assert lanes._staged_terminal_authority(staged, snapshot) is None + finally: + snapshot.close() + assert lanes._recover_retirement_staging() == 0 + assert not staged.exists() + assert len(list((board / "quarantine").glob("*.quarantine"))) == 1 + + +def test_noncanonical_pending_filename_is_quarantined_before_db_access( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + pending = board / "t_noncanonical.run-01.terminal.pending.json" + lanes.atomic_json( + pending, + _pending_terminal_record( + "cassandra", "t_noncanonical", 1, "noncanonical filename" + ), + ) + database = _NoDatabase() + monkeypatch.setitem( + sys.modules, + "hermes_cli", + type("HermesCli", (), {"kanban_db": database})(), + ) + + assert lanes.recover_pending_finalizations() == 0 + assert database.accesses == [] + assert not pending.exists() + assert len(list((board / "quarantine").glob("*.quarantine"))) == 1 + + +def test_noncanonical_prepared_filename_is_quarantined_before_db_access( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + prepared = ( + board / f"t_prepared.run-01.terminal.prepared-{'a' * 32}.json" + ) + record = _pending_terminal_record( + "cassandra", "t_prepared", 1, "noncanonical prepared filename" + ) + record["kanban_state"] = "prepared" + lanes.atomic_json(prepared, record) + database = _NoDatabase() + monkeypatch.setitem( + sys.modules, + "hermes_cli", + type("HermesCli", (), {"kanban_db": database})(), + ) + + assert lanes.recover_pending_finalizations() == 0 + assert database.accesses == [] + assert not prepared.exists() + assert len(list((board / "quarantine").glob("*.quarantine"))) == 1 + + +@pytest.mark.parametrize( + ("state", "suffix"), + [ + ("committed", "terminal.committed.json"), + ("conflict", f"terminal.conflict-{'a' * 32}.json"), + ], +) +def test_gc_quarantines_noncanonical_retained_evidence( + tmp_path: Path, + monkeypatch, + state: str, + suffix: str, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + artifact = board / f"t_retained.run-01.{suffix}" + record = _pending_terminal_record( + "cassandra", "t_retained", 1, "noncanonical retained filename" + ) + record["kanban_state"] = state + lanes.atomic_json(artifact, record) + + assert lanes.gc_lane_artifacts() == 1 + assert not artifact.exists() + assert len(list((board / "quarantine").glob("*.quarantine"))) == 1 diff --git a/testing/tests/test_hermes_cli_staging.py b/testing/tests/test_hermes_cli_staging.py new file mode 100644 index 00000000..a661b6c2 --- /dev/null +++ b/testing/tests/test_hermes_cli_staging.py @@ -0,0 +1,441 @@ +"""Retirement staging, recovery, and bounded quarantine.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + _completed_result, + _install_terminal_recovery_db, + _pending_terminal_record, + hashlib, + json, + lanes, + nullcontext, + os, + sys, + time, +) + + +def test_recovery_finalizes_the_valid_snapshot_it_classified_before_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path, first = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_valid_gap"), + board="cassandra", + task_id="t_valid_gap", + run_id=54, + structured=_completed_result("first valid snapshot"), + summary="first valid snapshot", + metadata={}, + ) + replacement = path.with_name("replacement.valid") + lanes.atomic_json( + replacement, + _pending_terminal_record( + "cassandra", "t_valid_gap", 54, "second valid replacement" + ), + ) + task = SimpleNamespace( + id="t_valid_gap", + status="running", + current_run_id=54, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + real_finalize = lanes._finalize_terminal_record + swapped = {"value": False} + + def replace_before_finalize(*args, **kwargs): + assert kwargs["snapshot"].document == first + if not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + return real_finalize(*args, **kwargs) + + monkeypatch.setattr(lanes, "_finalize_terminal_record", replace_before_finalize) + + assert lanes.recover_pending_finalizations() == 1 + assert task.result == first["result"] + assert completions == [first["result"]] + assert reclaims == [] + assert not path.exists() + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + conflict = json.loads(conflicts[0].read_text()) + assert conflict["summary"] == "second valid replacement" + + +def test_replacement_staged_journal_promotes_replays_and_never_reexecutes( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_staged"), 55) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed source") + source_stat = path.stat(follow_symlinks=False) + staged_winner = path.with_name("staged-winner.tmp") + canonical_replacement = path.with_name("canonical-replacement.tmp") + first = _pending_terminal_record( + "cassandra", "t_staged", 55, "staged accepted result" + ) + second = _pending_terminal_record( + "cassandra", "t_staged", 55, "later canonical replacement" + ) + lanes.atomic_json(staged_winner, first) + lanes.atomic_json(canonical_replacement, second) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path.parent, directory_flags) + real_rename = lanes._rename_noreplace + rename_steps = [] + + def force_replacement_staging(source, destination, **kwargs): + rename_steps.append((source, destination)) + if source == path.name: + os.replace(staged_winner, path) + elif source.startswith(".retire."): + os.replace(canonical_replacement, path) + return real_rename(source, destination, **kwargs) + + try: + with monkeypatch.context() as race: + race.setattr(lanes, "_rename_noreplace", force_replacement_staging) + outcome = lanes._retire_terminal_entry( + path, + source_stat, + board_descriptor=descriptor, + quarantine_descriptor=descriptor, + ) + finally: + os.close(descriptor) + + assert outcome == "replacement-staged" + assert len(rename_steps) == 2 + staged = list(path.parent.glob(".retire.*")) + assert len(staged) == 1 + staged_snapshot = lanes._open_terminal_recovery_snapshot(staged[0]) + assert staged_snapshot is not None + try: + authority = lanes._staged_terminal_authority(staged[0], staged_snapshot) + assert authority is not None + assert authority[0].run_id == 55 + assert authority[1] == path + finally: + staged_snapshot.close() + assert lanes._has_pending_finalization("cassandra", "t_staged", 55) is True + assert lanes.gc_lane_artifacts( + now=staged[0].stat().st_mtime + 3600, + max_age_seconds=0, + max_count=0, + max_bytes=0, + ) == 0 + assert staged[0].exists() + + task = SimpleNamespace( + id="t_staged", + status="running", + current_run_id=55, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + completions = [] + + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + task.status = "done" + task.result = kwargs["result"] + task.completed_run_id = kwargs["expected_run_id"] + task.current_run_id = None + completions.append(kwargs["result"]) + return True + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.recover_pending_finalizations() == 1 + assert completions == [first["result"]] + assert not list(path.parent.glob(".retire.*")) + assert not path.exists() + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + assert json.loads(conflicts[0].read_text())["result"] == second["result"] + assert lanes.recover_pending_finalizations() == 0 + assert completions == [first["result"]] + + +def test_quarantine_staged_replacement_is_recovered_bounded_and_preserved( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + path = lanes._terminal_path(lanes.state_path("cassandra", "t_qstage"), 58) + path.parent.mkdir(parents=True) + path.write_bytes(b"malformed original") + identity = lanes._terminal_identity(path) + assert identity is not None + records = [ + _pending_terminal_record( + "cassandra", "t_qstage", 58, f"quarantine replacement {number}" + ) + for number in range(2) + ] + replacements = [ + path.with_name(f"quarantine-replacement-{number}.tmp") + for number in range(2) + ] + for replacement, record in zip(replacements, records, strict=True): + lanes.atomic_json(replacement, record) + real_rename = lanes._rename_noreplace + retirement_steps = [] + + def replace_during_cross_directory_retirement(source, destination, **kwargs): + if source == path.name and destination.startswith(".retire."): + retirement_steps.append("move") + os.replace(replacements[0], path) + elif source.startswith(".retire.") and destination == path.name: + retirement_steps.append("restore") + os.replace(replacements[1], path) + return real_rename(source, destination, **kwargs) + + with monkeypatch.context() as race: + race.setattr( + lanes, + "_rename_noreplace", + replace_during_cross_directory_retirement, + ) + lanes._quarantine_terminal(path, identity, "malformed-payload") + + assert retirement_steps == ["move", "restore"] + quarantine_stages = list((path.parent / "quarantine").glob(".retire.*")) + assert len(quarantine_stages) == 1 + snapshot = lanes._open_terminal_recovery_snapshot(quarantine_stages[0]) + assert snapshot is not None + try: + authority = lanes._staged_terminal_authority(quarantine_stages[0], snapshot) + assert authority is not None + assert authority[0].run_id == 58 + assert authority[1] == path + finally: + snapshot.close() + assert lanes._has_pending_finalization("cassandra", "t_qstage", 58) is True + assert lanes.gc_lane_artifacts( + now=quarantine_stages[0].stat().st_mtime + 3600, + max_age_seconds=0, + max_count=0, + max_bytes=0, + ) == 1 + assert quarantine_stages[0].exists() + + task = SimpleNamespace( + id="t_qstage", + status="running", + current_run_id=58, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + + assert lanes.recover_pending_finalizations() == 1 + assert len(completions) == 1 + assert reclaims == [] + assert not list(path.parent.glob(".retire.*")) + assert not list((path.parent / "quarantine").glob(".retire.*")) + conflicts = list(path.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + preserved = {task.result, json.loads(conflicts[0].read_text())["result"]} + assert preserved == {record["result"] for record in records} + assert lanes.recover_pending_finalizations() == 0 + assert len(completions) == 1 + + +def test_repeated_retirement_staging_preserves_every_valid_result_once( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + canonical = lanes._terminal_path( + lanes.state_path("cassandra", "t_staged_repeat"), 57 + ) + canonical.parent.mkdir(parents=True) + path_digest = hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16] + staged = canonical.parent / f".retire.{path_digest}.{'a' * 16}.0" + records = [ + _pending_terminal_record( + "cassandra", "t_staged_repeat", 57, f"accepted result {number}" + ) + for number in range(3) + ] + lanes.atomic_json(staged, records[0]) + second = canonical.with_name("second-staged-replacement.tmp") + third = canonical.with_name("third-staged-replacement.tmp") + lanes.atomic_json(second, records[1]) + lanes.atomic_json(third, records[2]) + real_rename = lanes._rename_noreplace + retirement_steps = [] + + def replace_each_retirement_generation(source, destination, **kwargs): + if source == staged.name and destination.startswith(".retire."): + retirement_steps.append("move") + os.replace(second, staged) + elif source.startswith(".retire.") and destination == staged.name: + retirement_steps.append("restore") + os.replace(third, staged) + return real_rename(source, destination, **kwargs) + + with monkeypatch.context() as race: + race.setattr(lanes, "_rename_noreplace", replace_each_retirement_generation) + assert lanes._recover_retirement_staging() == 0 + + assert retirement_steps == ["move", "restore"] + staged_generations = list(canonical.parent.glob(".retire.*")) + assert len(staged_generations) == 2 + for generation in staged_generations: + snapshot = lanes._open_terminal_recovery_snapshot(generation) + assert snapshot is not None + try: + authority = lanes._staged_terminal_authority(generation, snapshot) + assert authority is not None + assert authority[1] == canonical + finally: + snapshot.close() + + task = SimpleNamespace( + id="t_staged_repeat", + status="running", + current_run_id=57, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + + assert lanes.recover_pending_finalizations() == 1 + assert len(completions) == 1 + assert reclaims == [] + assert not list(canonical.parent.glob(".retire.*")) + conflicts = list(canonical.parent.glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 2 + preserved_results = {task.result} + preserved_results.update(json.loads(path.read_text())["result"] for path in conflicts) + assert preserved_results == {record["result"] for record in records} + assert lanes.recover_pending_finalizations() == 0 + assert len(completions) == 1 + + +def test_prepared_recovery_quarantines_pinned_inode_then_replays_replacement( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + pending_identity = lanes.TerminalIdentity( + "cassandra", "t_prepared_swap", 59, "pending" + ) + pending = _pending_terminal_record( + "cassandra", "t_prepared_swap", 59, "valid prepared replacement" + ) + prepared_path = lanes._persist_prepared_evidence(pending_identity, pending) + valid_prepared = json.loads(prepared_path.read_text()) + replacement = prepared_path.with_name("prepared-valid-replacement.tmp") + lanes.atomic_json(replacement, valid_prepared) + invalid_prepared = dict(valid_prepared) + invalid_prepared["summary"] = "does not match the structured result" + lanes.atomic_json(prepared_path, invalid_prepared) + task = SimpleNamespace( + id="t_prepared_swap", + status="running", + current_run_id=59, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + completions = [] + reclaims = [] + _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) + real_quarantine = lanes._quarantine_terminal + swapped = {"value": False} + + def swap_before_prepared_quarantine(*args, **kwargs): + assert kwargs["snapshot"].document == invalid_prepared + if not swapped["value"]: + os.replace(replacement, prepared_path) + swapped["value"] = True + return real_quarantine(*args, **kwargs) + + monkeypatch.setattr( + lanes, + "_quarantine_terminal", + swap_before_prepared_quarantine, + ) + + assert lanes.recover_pending_finalizations() == 1 + assert swapped["value"] is True + assert completions == [valid_prepared["result"]] + assert reclaims == [] + assert not prepared_path.exists() + committed = list(prepared_path.parent.glob("*.terminal.committed.json")) + assert len(committed) == 1 + assert json.loads(committed[0].read_text())["result"] == valid_prepared["result"] + + +def test_untrusted_retirement_staging_is_bounded_by_artifact_gc( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + board = lanes.STATE_ROOT / "cassandra" + board.mkdir(parents=True) + for sequence in range(10): + (board / f".retire.{'a' * 16}.{'b' * 16}.{sequence}").write_bytes( + b"not terminal authority" + ) + (board / f".retire.{'c' * 32}.0").write_bytes(b"legacy hidden artifact") + + removed = lanes.gc_lane_artifacts( + now=time.time(), + max_age_seconds=3600, + max_count=2, + max_bytes=1024 * 1024, + ) + + assert removed == 9 + assert len(list(board.glob(".retire.*"))) == 2 + + +def test_untrusted_quarantine_stage_never_creates_recursive_quarantine( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + quarantine = lanes.STATE_ROOT / "cassandra" / "quarantine" + quarantine.mkdir(parents=True) + staged = quarantine / f".retire.{'a' * 16}.{'b' * 16}.0" + staged.write_bytes(b"untrusted staged bytes") + + assert lanes._recover_retirement_staging() == 0 + + assert not staged.exists() + assert not (quarantine / "quarantine").exists() + diagnostics = list(quarantine.glob("*.quarantine")) + assert len(diagnostics) == 1 + assert diagnostics[0].stat().st_mode & 0o777 == 0o600 diff --git a/testing/tests/test_hermes_cli_support.py b/testing/tests/test_hermes_cli_support.py new file mode 100644 index 00000000..5f34a37e --- /dev/null +++ b/testing/tests/test_hermes_cli_support.py @@ -0,0 +1,236 @@ +"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" + +from __future__ import annotations + +import errno + +import importlib.util + +import hashlib + +import json + +import os + +import signal + +import stat + +import sys + +import threading + +import time + +from contextlib import nullcontext + +from pathlib import Path + +from types import SimpleNamespace + +import pytest + +import yaml + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" + +sys.path.insert(0, str(SCRIPTS)) + +HERMES = Path(__file__).parents[2] / "services/hermes" + +KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" + +FLUX_HERMES = ( + Path(__file__).parents[2] + / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" +) + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + +_lane_facade = _load("cli_lane_runner") + + +class _LaneSurface: + """Patch every module-local import behind the compatibility facade.""" + + def __init__(self, facade): + object.__setattr__(self, "_facade", facade) + object.__setattr__( + self, + "_modules", + tuple( + module + for name, module in sys.modules.items() + if name == "cli_lane_runner" or name.startswith("cli_lane_") + ), + ) + + def __getattr__(self, name): + facade = object.__getattribute__(self, "_facade") + if hasattr(facade, name): + return getattr(facade, name) + for module in object.__getattribute__(self, "_modules"): + if hasattr(module, name): + return getattr(module, name) + raise AttributeError(name) + + def __setattr__(self, name, value): + matched = False + for module in object.__getattribute__(self, "_modules"): + if hasattr(module, name): + setattr(module, name, value) + matched = True + if not matched: + setattr(object.__getattribute__(self, "_facade"), name, value) + + +lanes = _LaneSurface(_lane_facade) + +policy = _load("claude_command_policy") + +migration = _load("migrate_herdr_state") + +auth_patch = _load("patch_hermes_auth") + +tui_gateway_patch = _load("patch_tui_gateway") + +codex_runtime_patch = _load("patch_codex_runtime") + +ttyd_patch = _load("patch_ttyd_index") + +client_config = _load("configure_agent_clients") + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" + +def _completed_result(summary: str = "done") -> dict: + return { + "status": "completed", + "summary": summary, + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + } + +def _pending_terminal_record(board: str, task_id: str, run_id: int, summary: str) -> dict: + structured = _completed_result(summary) + return { + "board": board, + "task_id": task_id, + "expected_run_id": run_id, + "result": json.dumps(structured, sort_keys=True), + "summary": summary, + "metadata": {}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + +def _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) -> None: + class Connection: + def close(self): + return None + + def complete_task(_conn, _task_id, **kwargs): + if task.status != "running" or task.current_run_id != kwargs["expected_run_id"]: + return False + task.status = "done" + task.result = kwargs["result"] + task.current_run_id = None + completions.append(kwargs["result"]) + return True + + def reclaim_task(_conn, _task_id, **kwargs): + reclaims.append(kwargs) + return False + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + complete_task=complete_task, + reclaim_task=reclaim_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + +__all__ = [ + 'FLUX_HERMES', + 'HERMES', + 'KEYCLOAK', + 'Path', + 'SCRIPTS', + 'SimpleNamespace', + '_SwitchyardResponse', + '_agent_deployment', + '_completed_result', + '_install_terminal_recovery_db', + '_load', + '_oauth_deployment', + '_pending_terminal_record', + '_services', + 'auth_patch', + 'client_config', + 'codex_runtime_patch', + 'errno', + 'hashlib', + 'importlib', + 'json', + 'lanes', + 'migration', + 'nullcontext', + 'os', + 'policy', + 'pytest', + 'signal', + 'stat', + 'sys', + 'threading', + 'time', + 'ttyd_patch', + 'tui_gateway_patch', + 'yaml', +] diff --git a/testing/tests/test_hermes_coordinator_boards.py b/testing/tests/test_hermes_coordinator_boards.py new file mode 100644 index 00000000..aa63943f --- /dev/null +++ b/testing/tests/test_hermes_coordinator_boards.py @@ -0,0 +1,204 @@ +"""Hermes coordinator board recovery contracts.""" + +from __future__ import annotations + +from test_hermes_coordinator_support import ( + Path, + _base_config, + coordinator, + pytest, + routing, + subprocess, + yaml, +) + + +def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): + (tmp_path / "config.yaml").write_text( + yaml.safe_dump(_base_config()), encoding="utf-8" + ) + (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") + codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") + claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) + monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") + + status = coordinator.refresh_once(tmp_path) + + status_path = tmp_path / "workspace/coordinator/model-routing.json" + assert status_path.is_file() + assert status["projects"]["cassandra"]["state"] == "ready" + assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} + assert "do-not-report" not in status_path.read_text(encoding="utf-8") + +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + +def test_cassandra_workspace_uses_valid_configured_worktree( + tmp_path: Path, monkeypatch +): + """The coordinator selects an explicit Git worktree but rejects stale paths.""" + active = tmp_path / "cassandra-v69" + active.mkdir() + (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) + + assert coordinator.cassandra_workspace() == active + + (active / ".git").unlink() + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + token_path = tmp_path / "gitea-token" + token_path.write_text("configured\n", encoding="utf-8") + monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + + def run(command, **kwargs): + commands.append(command) + environments.append(kwargs.get("env", {})) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + assert all("GITEA_TOKEN" not in environment for environment in environments) + +def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): + """Missing credentials skip only the fetch, not the local origin repair.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready; fetch skipped until Gitea token is configured" + assert [command[-3] for command in commands] == ["remote", "set-url"] + +def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): + """A project switch moves queued work without relocating active evidence.""" + + class Task: + def __init__(self, task_id: str, status: str): + self.id = task_id + self.status = status + self.workspace_kind = "dir" + self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) + + class Closing: + def __enter__(self): + return object() + + def __exit__(self, *_args): + return None + + class FakeKanban: + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] + moved: list[tuple[str, str]] = [] + + @staticmethod + def connect_closing(*, board: str): + assert board == "cassandra" + return Closing() + + @classmethod + def list_tasks(cls, _connection, *, include_archived: bool): + assert include_archived is False + return cls.tasks + + @classmethod + def set_workspace_path(cls, _connection, task_id: str, path: Path): + cls.moved.append((task_id, str(path))) + + active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") + coordinator._migrate_open_cassandra_tasks(FakeKanban, active) + + assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator_routing.py similarity index 54% rename from testing/tests/test_hermes_coordinator.py rename to testing/tests/test_hermes_coordinator_routing.py index bc2d5b5f..ec93322a 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator_routing.py @@ -1,44 +1,18 @@ -"""Unit tests for Hermes coordinator model routing and profile generation.""" +"""Hermes coordinator model routing contracts.""" from __future__ import annotations -import importlib.util -import json -import subprocess -import sys -import tomllib -from pathlib import Path - -import pytest -import yaml - - -SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -sys.path.insert(0, str(SCRIPT.parent)) -routing = importlib.import_module("hermes_model_routing") -catalog_resolver = importlib.import_module("routing_catalog") -SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) -assert SPEC and SPEC.loader -coordinator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = coordinator -SPEC.loader.exec_module(coordinator) - - -def _base_config() -> dict: - """Return the minimal coordinator configuration used by routing tests.""" - return { - "model": { - "provider": "openai-codex", - "default": "gpt-5.6-terra", - "model": "gpt-5.6-terra", - }, - "fallback_providers": [ - {"provider": "anthropic", "model": "claude-opus-5"}, - dict(routing.LOCAL_FALLBACK), - ], - "toolsets": ["kanban"], - "terminal": {"cwd": "/opt/data/workspace"}, - } +from test_hermes_coordinator_support import ( + Path, + SCRIPT, + _base_config, + catalog_resolver, + json, + routing, + subprocess, + tomllib, + yaml, +) def test_model_version_and_quality_selection_handle_new_and_small_models(): @@ -60,12 +34,10 @@ def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.choose_claude_for_effort(claude, "medium") == "claude-sonnet-5" assert routing.choose_claude_for_effort(claude, "xhigh") == "claude-opus-4.8" - def test_empty_catalog_retains_current_models(): assert routing.choose_codex_model([], "gpt-5.6-terra") == "gpt-5.6-terra" assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5" - def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): """Stable Switchyard aliases follow live releases and survive catalog outages.""" codex = routing.Catalog( @@ -112,7 +84,6 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): == current["providers"]["claude"]["tiers"] ) - def test_live_catalog_never_routes_to_a_removed_model_tier(): """A live provider catalog must replace a retired tier with a live model.""" previous = { @@ -144,7 +115,6 @@ def test_live_catalog_never_routes_to_a_removed_model_tier(): assert current["providers"]["codex"]["tiers"]["luna"] in codex.models assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6" - def test_switchyard_targets_have_unique_upstream_identities(): """Switchyard drops duplicate client/model pairs, so reject them in Git.""" manifest = yaml.safe_load( @@ -163,7 +133,6 @@ def test_switchyard_targets_have_unique_upstream_identities(): if route.get("target"): assert route["target"] in target_names - def test_worker_alias_resolves_before_cli_launch(): document = { "providers": { @@ -178,7 +147,6 @@ def test_worker_alias_resolves_before_cli_launch(): == "worker/codex/gpt-5.8-sol/high" ) - def test_codex_cli_login_counts_as_connected_runtime(monkeypatch): """AUTO routing must recognize the authenticated app-server CLI lane.""" monkeypatch.setattr(routing.shutil, "which", lambda name: "/usr/bin/codex") @@ -195,7 +163,6 @@ def test_codex_cli_login_counts_as_connected_runtime(monkeypatch): assert routing.codex_cli_authenticated() is True - def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): (tmp_path / "config.yaml").write_text( yaml.safe_dump(_base_config()), encoding="utf-8" @@ -270,7 +237,6 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): assert "claude-secret" not in json.dumps(routes) assert (tmp_path / "profiles/codex-high/.env").stat().st_mode & 0o777 == 0o600 - def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path): base = _base_config() base["model"]["model"] = base["model"]["default"] = "gpt-5.6-sol" @@ -286,7 +252,6 @@ def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path): assert current["model"]["provider"] == "atlas-switchyard" assert current["fallback_providers"] == [] - def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path): """A catalog outage must not bypass a managed worker's Switchyard route.""" (tmp_path / "config.yaml").write_text( @@ -311,200 +276,3 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path) ) assert worker["model"]["model"] == "atlas/manual/codex/sol" assert worker["model"]["provider"] == "atlas-switchyard" - - -def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): - (tmp_path / "config.yaml").write_text( - yaml.safe_dump(_base_config()), encoding="utf-8" - ) - (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") - codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") - claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") - monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) - monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) - monkeypatch.setattr( - coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} - ) - monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") - - status = coordinator.refresh_once(tmp_path) - - status_path = tmp_path / "workspace/coordinator/model-routing.json" - assert status_path.is_file() - assert status["projects"]["cassandra"]["state"] == "ready" - assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} - assert "do-not-report" not in status_path.read_text(encoding="utf-8") - - -def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( - tmp_path: Path, monkeypatch -): - """A damaged project board degrades Kanban instead of stopping all services.""" - - class FakeKanbanDbCorruptError(Exception): - def __init__(self): - super().__init__("integrity check failed") - self.db_path = tmp_path / "kanban.db" - self.backup_path = tmp_path / "kanban.db.corrupt.backup" - self.reason = "row out of order" - - def raise_corruption(_root: Path) -> None: - raise FakeKanbanDbCorruptError() - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - status = coordinator.bootstrap_cassandra_state(tmp_path) - - assert status == { - "state": "corrupt-preserved", - "database": str(tmp_path / "kanban.db"), - "backup": str(tmp_path / "kanban.db.corrupt.backup"), - "reason": "row out of order", - } - - -def test_unrelated_cassandra_bootstrap_failure_remains_fatal( - tmp_path: Path, monkeypatch -): - """Only the known fail-closed corruption state may be degraded.""" - - class FakeKanbanDbCorruptError(Exception): - pass - - def raise_permission_error(_root: Path) -> None: - raise PermissionError("cannot access board") - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - try: - coordinator.bootstrap_cassandra_state(tmp_path) - except PermissionError as error: - assert str(error) == "cannot access board" - else: - raise AssertionError("unrelated board errors must fail the coordinator refresh") - - -def test_cassandra_workspace_uses_valid_configured_worktree( - tmp_path: Path, monkeypatch -): - """The coordinator selects an explicit Git worktree but rejects stale paths.""" - active = tmp_path / "cassandra-v69" - active.mkdir() - (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) - - assert coordinator.cassandra_workspace() == active - - (active / ".git").unlink() - assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH - - -@pytest.mark.parametrize( - ("current_remote", "repair_action"), - [ - ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), - (None, "add"), - (coordinator.CASSANDRA_REMOTE, None), - ], - ids=["old-origin", "missing-origin", "canonical-origin"], -) -def test_cassandra_sync_repairs_existing_worktree( - tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None -): - """Existing worktrees use the canonical Atlas origin before fetching.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - token_path = tmp_path / "gitea-token" - token_path.write_text("configured\n", encoding="utf-8") - monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) - commands: list[list[str]] = [] - environments: list[dict[str, str]] = [] - - def run(command, **kwargs): - commands.append(command) - environments.append(kwargs.get("env", {})) - if command[-2:] == ["get-url", "origin"]: - return subprocess.CompletedProcess( - command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" - ) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" - repair_commands = [command for command in commands if repair_action in command] - assert bool(repair_commands) is bool(repair_action) - assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] - assert all("GITEA_TOKEN" not in environment for environment in environments) - - -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - commands: list[list[str]] = [] - - def run(command, **_kwargs): - commands.append(command) - stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None - return subprocess.CompletedProcess(command, 0, stdout=stdout) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - state = coordinator.sync_cassandra_repo({}) - - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] - - -def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): - """A project switch moves queued work without relocating active evidence.""" - - class Task: - def __init__(self, task_id: str, status: str): - self.id = task_id - self.status = status - self.workspace_kind = "dir" - self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) - - class Closing: - def __enter__(self): - return object() - - def __exit__(self, *_args): - return None - - class FakeKanban: - tasks = [ - Task("queued", "todo"), - Task("active", "running"), - Task("done", "done"), - ] - moved: list[tuple[str, str]] = [] - - @staticmethod - def connect_closing(*, board: str): - assert board == "cassandra" - return Closing() - - @classmethod - def list_tasks(cls, _connection, *, include_archived: bool): - assert include_archived is False - return cls.tasks - - @classmethod - def set_workspace_path(cls, _connection, task_id: str, path: Path): - cls.moved.append((task_id, str(path))) - - active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") - coordinator._migrate_open_cassandra_tasks(FakeKanban, active) - - assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py new file mode 100644 index 00000000..6f58ea5b --- /dev/null +++ b/testing/tests/test_hermes_coordinator_support.py @@ -0,0 +1,70 @@ +"""Unit tests for Hermes coordinator model routing and profile generation.""" + +from __future__ import annotations + +import importlib.util + +import json + +import subprocess + +import sys + +import tomllib + +from pathlib import Path + +import pytest + +import yaml + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" + +sys.path.insert(0, str(SCRIPT.parent)) + +routing = importlib.import_module("hermes_model_routing") + +catalog_resolver = importlib.import_module("routing_catalog") + +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) + +assert SPEC and SPEC.loader + +coordinator = importlib.util.module_from_spec(SPEC) + +sys.modules[SPEC.name] = coordinator + +SPEC.loader.exec_module(coordinator) + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } + +__all__ = ( + "Path", + "SCRIPT", + "SPEC", + "_base_config", + "catalog_resolver", + "coordinator", + "importlib", + "json", + "pytest", + "routing", + "subprocess", + "sys", + "tomllib", + "yaml", +) diff --git a/testing/tests/test_hermes_worker_recovery.py b/testing/tests/test_hermes_worker_recovery.py index 77307bd3..359bbc05 100644 --- a/testing/tests/test_hermes_worker_recovery.py +++ b/testing/tests/test_hermes_worker_recovery.py @@ -42,7 +42,7 @@ class FakeKanban: self.tasks = tasks self.has_board = board_exists self.connection = FakeConnection() - self.reclaimed: list[tuple[str, str]] = [] + self.reclaimed: list[tuple[str, str, int]] = [] self.comments: list[tuple[str, str, str]] = [] def board_exists(self, board: str) -> bool: @@ -61,9 +61,16 @@ class FakeKanban: assert connection is self.connection return self.tasks - def reclaim_task(self, connection, task_id: str, *, reason: str) -> bool: + def reclaim_task( + self, + connection, + task_id: str, + *, + reason: str, + expected_run_id: int, + ) -> bool: assert connection is self.connection - self.reclaimed.append((task_id, reason)) + self.reclaimed.append((task_id, reason, expected_run_id)) return task_id != "t_race" def add_comment(self, connection, task_id: str, author: str, body: str) -> None: @@ -75,16 +82,17 @@ def test_recover_running_tasks_requeues_only_claimed_workers() -> None: module = _load_module() kanban = FakeKanban( [ - SimpleNamespace(id="t_running", status="running"), + SimpleNamespace(id="t_running", status="running", current_run_id=10), SimpleNamespace(id="t_done", status="done"), - SimpleNamespace(id="t_race", status="running"), + SimpleNamespace(id="t_race", status="running", current_run_id=11), + SimpleNamespace(id="t_missing_run", status="running", current_run_id=None), ] ) assert module.recover_running_tasks(kanban) == ["t_running"] assert kanban.reclaimed == [ - ("t_running", module.REASON), - ("t_race", module.REASON), + ("t_running", module.REASON, 10), + ("t_race", module.REASON, 11), ] assert kanban.comments == [ ( From 762784da6b558eedce5ec3e3993fdf42e79a97ec Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 19:54:41 -0300 Subject: [PATCH 08/46] hermes: bound Atlas pull request client --- services/hermes/agent-configmap.yaml | 30 +- services/hermes/agent-deployment.yaml | 9 + services/hermes/kustomization.yaml | 7 + services/hermes/scripts/gitea_api.py | 422 ++++++++++++++---- .../manage-atlas-pull-requests/SKILL.md | 58 +++ .../agents/openai.yaml | 4 + testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 322 +++++++++++++ testing/tests/test_hermes_runtime_access.py | 40 +- 9 files changed, 784 insertions(+), 116 deletions(-) mode change 100644 => 100755 services/hermes/scripts/gitea_api.py create mode 100644 services/hermes/skills/manage-atlas-pull-requests/SKILL.md create mode 100644 services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml create mode 100644 testing/tests/test_hermes_gitea_pr_client.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 0b81379d..b35da59a 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -315,15 +315,21 @@ data: real branch only when the requested implementation is review-ready. Never force-push. - `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh` - skill for an Atlas remote, and do not interpret an unauthenticated Gitea - HTTP 404 as a missing private repository. Use authenticated Git for clone, - fetch, and push. For pull-request metadata, comments, and merges, call - `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime - Vault token without exposing it to the command line, environment, output, - or transcript. Before a consequential merge, independently verify the - exact base/head ancestry and diff, run the relevant tests, and confirm the - candidate Jenkins result. Prefer the internal endpoint in + `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a + GitHub/`gh` skill for an Atlas remote, and do not interpret an + unauthenticated Gitea HTTP 404 as a missing private repository. Use + authenticated Git for clone, fetch, and non-force push. For repository and + pull-request reads, or to create/update a review-ready draft PR, load + `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The + client injects the runtime Vault token without exposing it to the command + line, environment, output, or transcript. It permits only Atlas-scoped + reads, same-repository draft creation, and title/body updates while the PR + remains draft. Merge, approve, close, delete, comments, repository + administration, and force-push are unavailable; never bypass the client + with raw HTTP. Leave every PR unmerged for Brad's review. Independently + verify the exact base/head ancestry and diff, run the relevant tests, and + confirm the candidate Jenkins result before presenting a handoff. Prefer + the internal endpoint in `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the existing cluster access to inspect the controller's job/build files and logs rather than guessing a public hostname. The preferred read-only path @@ -331,10 +337,8 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe - `--field KEY=VALUE` arguments over hand-quoted JSON. - After merging, observe the default-branch build to a terminal result and - report exact commit, build, and test evidence. + multibranch name. After Brad merges, observe the default-branch build to a + terminal result and report exact commit, build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..4b93f92d 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -663,6 +663,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -826,6 +827,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -894,6 +896,7 @@ spec: - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1188,6 +1191,12 @@ spec: - {key: dashboard-api.py, path: dashboard/plugin_api.py} - {key: dashboard-index.js, path: dashboard/dist/index.js} - {key: dashboard-style.css, path: dashboard/dist/style.css} + - name: atlas-pr-skill + configMap: + name: hermes-atlas-pr-skill + items: + - {key: SKILL.md, path: SKILL.md} + - {key: openai.yaml, path: agents/openai.yaml} - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..87984530 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -158,3 +158,10 @@ configMapGenerator: - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md options: disableNameSuffixHash: true + - name: hermes-atlas-pr-skill + namespace: hermes + files: + - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md + - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py old mode 100644 new mode 100755 index 5544c398..73a2bffd --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -1,43 +1,158 @@ #!/usr/bin/env python3 -"""Call the private Atlas Gitea API through a runtime-only token boundary.""" +"""Use the private Atlas Forgejo API through a least-authority PR boundary.""" from __future__ import annotations import argparse import json import os +import re import sys import urllib.error import urllib.parse import urllib.request +from collections.abc import Callable from pathlib import Path - -DEFAULT_BASE_URL = "https://scm.bstein.dev" +CANONICAL_BASE_URL = "https://scm.bstein.dev" +ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") +READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") +SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") +MAX_ERROR_BYTES = 8192 +DRAFT_TITLE_PREFIX = "WIP: " + + +class PolicyError(ValueError): + """Raised when a requested Forgejo operation exceeds the safe boundary.""" def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: - """Read and validate the token from its in-memory Vault projection.""" + """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() if not token: raise ValueError(f"runtime credential is empty: {path}") return token -def api_url(base_url: str, path: str) -> str: - """Return a same-origin Gitea API URL for a validated API path.""" - base = urllib.parse.urlsplit(base_url.rstrip("/")) +def configured_base_url() -> str: + """Reject attempts to redirect credentials away from the canonical origin.""" + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + if configured != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + return configured + + +def _validate_repo(repo: str) -> str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _validate_ref(value: object, name: str) -> str: + if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + raise PolicyError(f"{name} must be a same-repository branch name") + if ( + value.startswith(".") + or value.endswith(("/", ".", ".lock")) + or "//" in value + or ".." in value + or "@{" in value + ): + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + if len(value) > maximum or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _draft_title(value: object) -> str: + """Return a bounded title using Gitea's configured default draft prefix.""" + title = _validate_text(value, "title", 251, required=True).strip() + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _split_api_path(path: str) -> urllib.parse.SplitResult: target = urllib.parse.urlsplit(path) - if base.scheme not in {"http", "https"} or not base.netloc: - raise ValueError("GITEA_BASE_URL must be an absolute HTTP(S) URL") if target.scheme or target.netloc or target.fragment: - raise ValueError("API path must be relative to the configured Gitea origin") + raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): - raise ValueError("API path must start with /api/v1/") + raise PolicyError("API path must start with /api/v1/") + if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + raise PolicyError("encoded or non-canonical API paths are not allowed") + return target + + +def api_url(base_url: str, path: str) -> str: + """Return a canonical same-origin URL without forwarding credentials.""" + if base_url.rstrip("/") != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + target = _split_api_path(path) return urllib.parse.urlunsplit( - (base.scheme, base.netloc, target.path, target.query, "") + ("https", "scm.bstein.dev", target.path, target.query, "") + ) + + +def authorize_request(method: str, path: str, data: object | None) -> str: + """Validate one request and return its bounded operation name.""" + normalized_method = method.upper() + target = _split_api_path(path) + if normalized_method == "GET": + if data is not None: + raise PolicyError("read operations cannot include a request body") + match = READ_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("reads are limited to repositories owned by atlas") + _validate_repo(match.group(1)) + return "read" + + if target.query: + raise PolicyError("mutating operations cannot include query parameters") + if normalized_method == "POST": + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_text(data["body"], "body", 65536, required=False) + return "create-draft" + + if normalized_method == "PATCH": + match = PR_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: + raise PolicyError("draft updates accept only title and body") + if "title" in data and data["title"] != _draft_title(data["title"]): + raise PolicyError("updated pull-request titles must preserve draft state") + if "body" in data: + _validate_text(data["body"], "body", 65536, required=False) + return "update-draft" + + raise PolicyError( + "method is not available: merge, approve, close, delete, and force-push are outside this client" ) @@ -49,100 +164,255 @@ def build_request( token: str, data: object | None = None, ) -> urllib.request.Request: - """Build one authenticated request without placing the token in its URL.""" + """Build an authorized request without putting the token in its URL or body.""" + authorize_request(method, path, data) payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") return urllib.request.Request( api_url(base_url, path), data=payload, - method=method, + method=method.upper(), headers={ "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-operator/1", + "User-Agent": "hermes-atlas-pr-client/1", }, ) -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse a bounded method, API path, and optional JSON request body.""" - parser = argparse.ArgumentParser( - description="Call the Atlas Gitea API using the runtime Vault token." +def redact_bytes(value: bytes, token: str) -> bytes: + """Remove exact runtime-token and authorization-header values from output.""" + redacted = value.replace(token.encode("utf-8"), b"[REDACTED]") + return re.sub( + rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+", + rb"\1[REDACTED]", + redacted, ) - parser.add_argument("method", choices=ALLOWED_METHODS) - parser.add_argument("path", help="Gitea path beginning with /api/v1/") - data_group = parser.add_mutually_exclusive_group() - data_group.add_argument("--data-json", help="JSON object/array request body") - data_group.add_argument( - "--data-file", type=Path, help="path to a JSON request body" - ) - data_group.add_argument( - "--field", - action="append", - metavar="KEY=VALUE", - help=( - "repeatable JSON field; bare text remains a string while true, false, " - "null, numbers, objects, and arrays are decoded as JSON" - ), - ) - return parser.parse_args(argv) -def parse_fields(values: list[str]) -> dict[str, object]: - """Build a JSON object from shell-safe, repeatable key/value arguments.""" - result: dict[str, object] = {} - for value in values: - key, separator, raw = value.partition("=") - if not separator or not key or any(char.isspace() for char in key): - raise ValueError("each --field must be KEY=VALUE with a non-space key") - try: - result[key] = json.loads(raw) - except json.JSONDecodeError: - result[key] = raw +def _request( + method: str, + path: str, + data: object | None, + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + request = build_request( + method, + path, + base_url=configured_base_url(), + token=token, + data=data, + ) + with opener(request, timeout=30) as response: # type: ignore[attr-defined] + return redact_bytes(response.read(), token) # type: ignore[attr-defined] + + +def read( + path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen +) -> bytes: + """Read one Atlas repository API resource.""" + return _request("GET", path, None, token=token, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + title: str, + body: str, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Create a same-repository draft pull request for human review.""" + data = { + "base": base, + "body": body, + "head": head, + "title": _draft_title(title), + } + result = _request( + "POST", + f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + data, + token=token, + opener=opener, + ) + _require_draft_response(result) return result -def load_data(args: argparse.Namespace) -> object | None: - """Decode the optional JSON body without involving a shell expansion.""" - if args.data_json is not None: - return json.loads(args.data_json) - if args.data_file is not None: - return json.loads(args.data_file.read_text(encoding="utf-8")) - if args.field is not None: - return parse_fields(args.field) - return None +def _require_draft_response(value: bytes) -> dict[str, object]: + """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict) or document.get("draft") is not True: + raise PolicyError( + "Forgejo did not confirm draft state; human review is required" + ) + return document + + +def update_draft( + repo: str, + number: int, + updates: dict[str, str], + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Update title/body only after Forgejo confirms the PR is still a draft.""" + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("pull-request number must be positive") + path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" + current_raw = _request("GET", path, None, token=token, opener=opener) + try: + current = json.loads(current_raw) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(current, dict) or current.get("draft") is not True: + raise PolicyError("pull request is not a draft; human review now owns it") + safe_updates = dict(updates) + if "title" in safe_updates: + safe_updates["title"] = _draft_title(safe_updates["title"]) + result = _request("PATCH", path, safe_updates, token=token, opener=opener) + _require_draft_response(result) + return result + + +def _body(args: argparse.Namespace) -> str: + if args.body_file is None: + return "" + return args.body_file.read_text(encoding="utf-8") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the three intentionally narrow client operations.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and describe the request without reading a token or using the network", + ) + subparsers = parser.add_subparsers(dest="operation", required=True) + + read_parser = subparsers.add_parser( + "read", help="read an Atlas repository API path" + ) + read_parser.add_argument("path") + + create_parser = subparsers.add_parser( + "create-draft", help="create a same-repository draft pull request" + ) + create_parser.add_argument("repo") + create_parser.add_argument("--base", required=True) + create_parser.add_argument("--head", required=True) + create_parser.add_argument("--title", required=True) + create_parser.add_argument("--body-file", type=Path) + + update_parser = subparsers.add_parser( + "update-draft", help="update title/body of an existing draft pull request" + ) + update_parser.add_argument("repo") + update_parser.add_argument("number", type=int) + update_parser.add_argument("--title") + update_parser.add_argument("--body-file", type=Path) + return parser.parse_args(argv) + + +def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: + if args.operation == "read": + return "GET", args.path, None + repo = _validate_repo(args.repo) + if args.operation == "create-draft": + data = { + "base": args.base, + "body": _body(args), + "head": args.head, + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data + if args.number < 1: + raise PolicyError("pull-request number must be positive") + data = {} + if args.title is not None: + data["title"] = _draft_title(args.title) + if args.body_file is not None: + data["body"] = _body(args) + return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + + +def _write_body(body: bytes) -> None: + if body: + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.buffer.write(b"\n") def main(argv: list[str] | None = None) -> int: - """Execute the request, print only its response body, and return HTTP status.""" - args = parse_args(argv) + """Validate, optionally describe, and execute one safe Forgejo operation.""" + token = "" try: - request = build_request( - args.method, - args.path, - base_url=os.environ.get("GITEA_BASE_URL", DEFAULT_BASE_URL), - token=read_token(), - data=load_data(args), - ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read() - if body: - sys.stdout.buffer.write(body) - if not body.endswith(b"\n"): - sys.stdout.buffer.write(b"\n") + args = parse_args(argv) + method, path, data = _operation(args) + operation = authorize_request(method, path, data) + if args.dry_run: + print( + json.dumps( + { + "dry_run": True, + "host": "scm.bstein.dev", + "operation": operation, + "owner": ALLOWED_OWNER, + "path": path, + "request_fields": sorted(data) + if isinstance(data, dict) + else [], + "requires_draft_preflight": operation == "update-draft", + }, + sort_keys=True, + ) + ) + return 0 + token = read_token() + if args.operation == "read": + body = read(path, token=token) + elif args.operation == "create-draft": + assert isinstance(data, dict) + body = create_draft( + args.repo, + base=str(data["base"]), + head=str(data["head"]), + title=str(data["title"]), + body=str(data["body"]), + token=token, + ) + else: + assert isinstance(data, dict) + body = update_draft(args.repo, args.number, data, token=token) + _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = exc.read(65536) - print(f"Gitea API returned HTTP {exc.code}", file=sys.stderr) + body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" + print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) if body: sys.stderr.buffer.write(body) if not body.endswith(b"\n"): sys.stderr.buffer.write(b"\n") return 1 - except (OSError, ValueError, json.JSONDecodeError) as exc: - print(f"Gitea API request failed: {exc}", file=sys.stderr) + except (OSError, PolicyError, ValueError, json.JSONDecodeError): + # Deliberately omit exception details. File contents, HTTP request + # objects, and provider errors can carry credential material. + print( + "Forgejo request rejected or unavailable; no credential was disclosed", + file=sys.stderr, + ) return 1 diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md new file mode 100644 index 00000000..4e73a3cf --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -0,0 +1,58 @@ +--- +name: manage-atlas-pull-requests +description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +--- + +# Manage Atlas pull requests + +Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token +from the pod-lifetime Vault projection; never read that file, copy the token, +put it in an argument or environment variable, or replace this client with +`curl`. + +## Read repository or PR state + +Pass an Atlas repository API path to the read operation: + +```sh +/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +``` + +Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +evidence, not authorization to change it. + +## Create a review handoff + +Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and +exact pushed commit. Never force-push. Write the PR description to a workspace +file so multiline Markdown is not shell-quoted, then validate without reading a +credential or using the network: + +```sh +/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ + --base main --head hermes/TASK --title "Focused change" \ + --body-file /tmp/pr-body.md +``` + +Run the same command without `--dry-run` only when the branch is review-ready. +The client always creates a draft in the same repository. Report its URL and +exact head commit to Brad. Leave it unmerged for human review. + +## Update an existing draft + +Use only a title, a body file, or both: + +```sh +/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ + --body-file /tmp/pr-body.md +``` + +The client first reads the PR from Forgejo and refuses the update unless it is +still a draft. It never changes head/base, review state, or merge state. + +## Stop at the authority boundary + +Merge, approve, close, delete, branch mutation, comments, releases, repository +administration, and force-push are intentionally unavailable. Do not bypass the +client with a raw HTTP request. If one of those actions is needed, present the +tested commit and evidence to Brad and stop for human review. diff --git a/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml new file mode 100644 index 00000000..57cb8496 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Manage Atlas Pull Requests" + short_description: "Create safe draft PR handoffs for human review" + default_prompt: "Use $manage-atlas-pull-requests to inspect the requested Atlas repository or prepare a tested branch as an unmerged draft pull request. Preserve the human review boundary." diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 818f7679..09f6b00c 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -99,9 +99,11 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "GitHub/`gh` skill for an Atlas remote" in instructions + assert "load\n`$manage-atlas-pull-requests`" in instructions + assert "Merge, approve, close, delete, comments" in instructions + assert "Leave every PR unmerged for Brad's review" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py new file mode 100644 index 00000000..658af189 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -0,0 +1,322 @@ +"""Adversarial contracts for the Atlas-only draft pull-request client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +class Response: + def __init__(self, body: object): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return self.body + + +@pytest.mark.parametrize( + ("base_url", "path"), + [ + ("https://evil.example", "/api/v1/repos/atlas/cassandra"), + ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/atlas/../admin"), + ], +) +def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request("GET", path, base_url=base_url, token="secret") + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_update_checks_server_draft_state_before_patching(): + client = _load() + calls = [] + responses = iter( + [ + Response({"number": 7, "draft": True}), + Response({"number": 7, "draft": True}), + ] + ) + + def opener(request, timeout): + calls.append((request.method, request.full_url, request.data, timeout)) + return next(responses) + + result = client.update_draft( + "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener + ) + + assert json.loads(result) == {"number": 7, "draft": True} + assert [call[0] for call in calls] == ["GET", "PATCH"] + assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} + + +def test_update_rejects_non_draft_without_sending_patch(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request.method, timeout)) + return Response({"number": 7, "draft": False}) + + with pytest.raises(client.PolicyError, match="human review now owns"): + client.update_draft( + "cassandra", 7, {"body": "new"}, token="runtime", opener=opener + ) + assert calls == [("GET", 30)] + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" + + +def test_create_uses_live_gitea_schema_and_verifies_draft_response(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((json.loads(request.data), timeout)) + return Response({"number": 3, "draft": True}) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result)["draft"] is True + assert calls[0][0]["title"] == "WIP: Focused fix" + assert "draft" not in calls[0][0] + + +def test_create_fails_closed_when_server_does_not_confirm_draft(): + client = _load() + + with pytest.raises(client.PolicyError, match="did not confirm draft"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +def test_dry_run_does_not_read_token_or_use_network( + tmp_path: Path, monkeypatch, capsys +): + client = _load() + body = tmp_path / "body.md" + body.write_text("Evidence only\n", encoding="utf-8") + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--title", + "Repair", + "--body-file", + str(body), + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..7ad5087a 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -11,7 +11,6 @@ from pathlib import Path import pytest import yaml - ROOT = Path(__file__).parents[2] HERMES = ROOT / "services" / "hermes" SCRIPTS = HERMES / "scripts" @@ -31,19 +30,23 @@ def test_gitea_api_builds_runtime_authenticated_same_origin_requests(): request = gitea_api.build_request( "POST", - "/api/v1/repos/atlas/cassandra/issues/1/comments", + "/api/v1/repos/atlas/cassandra/pulls", base_url="https://scm.bstein.dev", token="runtime-only-value", - data={"body": "reviewed"}, + data={ + "base": "main", + "body": "Ready for review.", + "head": "hermes/repair", + "title": "WIP: Repair semantic review findings", + }, ) assert isinstance(request, urllib.request.Request) - assert request.full_url == ( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/issues/1/comments" - ) + assert request.full_url == "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls" assert request.method == "POST" assert request.get_header("Authorization") == "token runtime-only-value" - assert json.loads(request.data) == {"body": "reviewed"} + assert json.loads(request.data)["title"].startswith("WIP: ") + assert "draft" not in json.loads(request.data) @pytest.mark.parametrize( @@ -61,25 +64,14 @@ def test_gitea_api_rejects_foreign_or_non_api_targets(path: str): gitea_api.api_url("https://scm.bstein.dev", path) -def test_gitea_api_parses_shell_safe_fields_as_json_values(): +@pytest.mark.parametrize("method", ["DELETE", "PUT", "OPTIONS"]) +def test_gitea_api_rejects_unavailable_methods(method: str): gitea_api = _load("gitea_api") - assert gitea_api.parse_fields( - ["Do=manually-merged", "MergeCommitID=abc123", "enabled=true", "count=2"] - ) == { - "Do": "manually-merged", - "MergeCommitID": "abc123", - "enabled": True, - "count": 2, - } - - -@pytest.mark.parametrize("value", ["missing-separator", "=missing-key", "bad key=x"]) -def test_gitea_api_rejects_invalid_shell_safe_fields(value: str): - gitea_api = _load("gitea_api") - - with pytest.raises(ValueError): - gitea_api.parse_fields([value]) + with pytest.raises(gitea_api.PolicyError): + gitea_api.authorize_request( + method, "/api/v1/repos/atlas/cassandra/pulls/1", None + ) def test_jenkins_evidence_ignores_nested_execution_result(): From cf3e46c2b684270ff931d82fcadcee6777fb9543 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:18:49 -0300 Subject: [PATCH 09/46] hermes: harden Forgejo client boundary --- services/hermes/agent-configmap.yaml | 33 +- services/hermes/scripts/gitea_api.py | 381 +++++++++------- .../manage-atlas-pull-requests/SKILL.md | 48 +- testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 417 +++++++++++------- .../tests/test_hermes_gitea_pr_integration.py | 147 ++++++ 6 files changed, 671 insertions(+), 363 deletions(-) create mode 100644 testing/tests/test_hermes_gitea_pr_integration.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index b35da59a..782876cf 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -318,15 +318,15 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For repository and - pull-request reads, or to create/update a review-ready draft PR, load + authenticated Git for clone, fetch, and non-force push. For bounded + repository/pull-request evidence or to create a review-ready draft PR, load `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The client injects the runtime Vault token without exposing it to the command line, environment, output, or transcript. It permits only Atlas-scoped - reads, same-repository draft creation, and title/body updates while the PR - remains draft. Merge, approve, close, delete, comments, repository - administration, and force-push are unavailable; never bypass the client - with raw HTTP. Leave every PR unmerged for Brad's review. Independently + metadata reads and verified same-repository draft creation. Updates, merge, + approve, close, delete, comments, repository administration, and force-push + are unavailable; never bypass the client with raw HTTP. Leave every PR + unmerged for Brad's review. Independently verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in @@ -337,25 +337,24 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. After Brad merges, observe the default-branch build to a - terminal result and report exact commit, build, and test evidence. + multibranch name. If Brad separately reports that he merged the PR, observe + the default-branch build to a terminal result and report exact commit, + build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a clean branch switch when comparing revisions, and preserve any unexpected - file as user state. Use the Gitea pull-request merge endpoint for an open - PR so both Git history and PR state remain auditable. If an authorized - manual merge already reached the base branch, reconcile the open PR with - `Do=manually-merged` and its exact merge commit instead of leaving stale - review state. + file as user state. PR publication ends at the verified open draft; Brad + owns review and merge authority. The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the relevant `titan-iac` manifests. Put durable desired-state changes on a - reviewable `titan-iac` branch, validate Kustomize and client dry-run, then - use Flux reconciliation after the tracked change is published. Direct - `kubectl` mutations are for explicit incident recovery or ephemeral - verification, not ordinary delivery. + reviewable `titan-iac` branch and validate Kustomize and client dry-run. + Brad owns merge and routine Flux reconciliation; perform either only under + a separate explicit operator request. Direct `kubectl` mutations are for + explicit incident recovery or ephemeral verification, not ordinary + delivery. Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned and password fallback diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 73a2bffd..057a79a1 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -7,6 +7,7 @@ import argparse import json import os import re +import subprocess import sys import urllib.error import urllib.parse @@ -18,18 +19,41 @@ CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") -READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") -SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") MAX_ERROR_BYTES = 8192 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +SENSITIVE_BODY_PATTERNS = ( + re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), + re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), + re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), + re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), +) class PolicyError(ValueError): """Raised when a requested Forgejo operation exceeds the safe boundary.""" +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Reject every redirect before urllib can copy authentication headers.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("Forgejo redirects are not allowed") + + +_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _safe_urlopen(request: urllib.request.Request, timeout: int): + """Open one request using a handler that never follows redirects.""" + return _SAFE_OPENER.open(request, timeout=timeout) + + def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() @@ -53,19 +77,29 @@ def _validate_repo(repo: str) -> str: def _validate_ref(value: object, name: str) -> str: - if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + """Validate the complete Git ref grammar using Git itself.""" + if not isinstance(value, str) or not value or value.startswith("-"): raise PolicyError(f"{name} must be a same-repository branch name") - if ( - value.startswith(".") - or value.endswith(("/", ".", ".lock")) - or "//" in value - or ".." in value - or "@{" in value - ): + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: raise PolicyError(f"{name} must be a safe same-repository branch name") return value +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: if not isinstance(value, str): raise PolicyError(f"{name} must be text") @@ -76,6 +110,15 @@ def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> return value +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, required=False) + if any(secret and secret in body for secret in forbidden): + raise PolicyError("pull-request body contains runtime credential material") + if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): + raise PolicyError("pull-request body resembles credential material") + return body + + def _draft_title(value: object) -> str: """Return a bounded title using Gitea's configured default draft prefix.""" title = _validate_text(value, "title", 251, required=True).strip() @@ -99,6 +142,73 @@ def _split_api_path(path: str) -> urllib.parse.SplitResult: return target +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + try: + pairs = urllib.parse.parse_qsl( + target.query, keep_blank_values=True, strict_parsing=True + ) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not values[name].isdigit() or int(values[name]) < 1: + raise PolicyError(f"{name} must be a positive integer") + if "limit" in values and int(values["limit"]) > 50: + raise PolicyError("read limit cannot exceed 50") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") + + +def _authorize_read(target: urllib.parse.SplitResult) -> str: + """Allow only repository, PR, branch, commit, and status metadata reads.""" + prefix = "/api/v1/repos/atlas/" + remainder = target.path.removeprefix(prefix) + if remainder == target.path: + raise PolicyError("reads are limited to explicit Atlas repository metadata") + repo, separator, suffix = remainder.partition("/") + _validate_repo(repo) + + if not separator: + _validate_query(target, set()) + return "repository" + if suffix == "pulls": + _validate_query(target, {"page", "limit", "state"}) + return "pull-list" + if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + _validate_query(target, set()) + return "pull" + if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + _validate_query(target, {"page", "limit"}) + return "pull-evidence" + if suffix == "branches": + _validate_query(target, {"page", "limit"}) + return "branch-list" + branch_match = re.fullmatch(r"branches/([^/]+)", suffix) + if branch_match: + _validate_ref(branch_match.group(1), "branch") + _validate_query(target, set()) + return "branch" + if suffix == "commits": + _validate_query(target, {"page", "limit"}) + return "commit-list" + if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix): + _validate_query(target, set()) + return "commit" + if re.fullmatch( + r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})", + suffix, + ): + _validate_query(target, {"page", "limit"}) + return "status" + raise PolicyError("repository API route is outside the metadata read allowlist") + + def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" if base_url.rstrip("/") != CANONICAL_BASE_URL: @@ -116,44 +226,23 @@ def authorize_request(method: str, path: str, data: object | None) -> str: if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - match = READ_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("reads are limited to repositories owned by atlas") - _validate_repo(match.group(1)) - return "read" - + return _authorize_read(target) if target.query: raise PolicyError("mutating operations cannot include query parameters") - if normalized_method == "POST": - match = CREATE_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("POST is limited to creating an Atlas draft pull request") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: - raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): - raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_text(data["body"], "body", 65536, required=False) - return "create-draft" - - if normalized_method == "PATCH": - match = PR_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: - raise PolicyError("draft updates accept only title and body") - if "title" in data and data["title"] != _draft_title(data["title"]): - raise PolicyError("updated pull-request titles must preserve draft state") - if "body" in data: - _validate_text(data["body"], "body", 65536, required=False) - return "update-draft" - - raise PolicyError( - "method is not available: merge, approve, close, delete, and force-push are outside this client" - ) + if normalized_method != "POST": + raise PolicyError("only read and create-draft operations are available") + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_body(data["body"]) + return "create-draft" def build_request( @@ -177,7 +266,7 @@ def build_request( "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-pr-client/1", + "User-Agent": "hermes-atlas-pr-client/2", }, ) @@ -198,102 +287,116 @@ def _request( data: object | None, *, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: request = build_request( - method, - path, - base_url=configured_base_url(), - token=token, - data=data, + method, path, base_url=configured_base_url(), token=token, data=data ) with opener(request, timeout=30) as response: # type: ignore[attr-defined] - return redact_bytes(response.read(), token) # type: ignore[attr-defined] + body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(body) > MAX_RESPONSE_BYTES: + raise PolicyError("Forgejo response exceeds the safe size limit") + return redact_bytes(body, token) def read( - path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen + path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: - """Read one Atlas repository API resource.""" + """Read one explicitly allowed Atlas repository metadata resource.""" return _request("GET", path, None, token=token, opener=opener) +def _nested(document: dict[str, object], *keys: str) -> object: + value: object = document + for key in keys: + if not isinstance(value, dict) or key not in value: + raise PolicyError("Forgejo omitted required pull-request evidence") + value = value[key] + return value + + +def _require_create_response( + value: bytes, + *, + repo: str, + base: str, + head: str, + head_sha: str, + title: str, + body: str, +) -> dict[str, object]: + """Require exact server evidence for the intended open draft handoff.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict): + raise PolicyError("Forgejo returned invalid pull-request metadata") + number = document.get("number") + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("Forgejo omitted a valid pull-request number") + full_name = f"{ALLOWED_OWNER}/{repo}" + expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" + expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" + checks = ( + document.get("state") == "open", + document.get("draft") is True, + document.get("merged") is False, + document.get("html_url") == expected_html, + document.get("url") == expected_api, + document.get("title") == title, + document.get("body") == body, + _nested(document, "base", "ref") == base, + _nested(document, "base", "repo", "full_name") == full_name, + _nested(document, "head", "ref") == head, + _nested(document, "head", "sha") == head_sha, + _nested(document, "head", "repo", "full_name") == full_name, + ) + if not all(checks): + raise PolicyError("Forgejo did not confirm the exact draft pull-request state") + return document + + def create_draft( repo: str, *, base: str, head: str, + head_sha: str, title: str, body: str, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: - """Create a same-repository draft pull request for human review.""" - data = { - "base": base, - "body": body, - "head": head, - "title": _draft_title(title), - } + """Create and verify a same-repository draft pull request for human review.""" + repo = _validate_repo(repo) + base = _validate_ref(base, "base") + head = _validate_ref(head, "head") + head_sha = _validate_sha(head_sha) + title = _draft_title(title) + body = _validate_body(body, forbidden=(token,)) + data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", - f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data, token=token, opener=opener, ) - _require_draft_response(result) + _require_create_response( + result, + repo=repo, + base=base, + head=head, + head_sha=head_sha, + title=title, + body=body, + ) return result -def _require_draft_response(value: bytes) -> dict[str, object]: - """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" - try: - document = json.loads(value) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(document, dict) or document.get("draft") is not True: - raise PolicyError( - "Forgejo did not confirm draft state; human review is required" - ) - return document - - -def update_draft( - repo: str, - number: int, - updates: dict[str, str], - *, - token: str, - opener: Callable[..., object] = urllib.request.urlopen, -) -> bytes: - """Update title/body only after Forgejo confirms the PR is still a draft.""" - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("pull-request number must be positive") - path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" - current_raw = _request("GET", path, None, token=token, opener=opener) - try: - current = json.loads(current_raw) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(current, dict) or current.get("draft") is not True: - raise PolicyError("pull request is not a draft; human review now owns it") - safe_updates = dict(updates) - if "title" in safe_updates: - safe_updates["title"] = _draft_title(safe_updates["title"]) - result = _request("PATCH", path, safe_updates, token=token, opener=opener) - _require_draft_response(result) - return result - - -def _body(args: argparse.Namespace) -> str: - if args.body_file is None: - return "" - return args.body_file.read_text(encoding="utf-8") - - def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse the three intentionally narrow client operations.""" + """Parse the intentionally narrow read and create-draft operations.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", @@ -301,28 +404,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="validate and describe the request without reading a token or using the network", ) subparsers = parser.add_subparsers(dest="operation", required=True) - - read_parser = subparsers.add_parser( - "read", help="read an Atlas repository API path" - ) + read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata") read_parser.add_argument("path") - create_parser = subparsers.add_parser( "create-draft", help="create a same-repository draft pull request" ) create_parser.add_argument("repo") create_parser.add_argument("--base", required=True) create_parser.add_argument("--head", required=True) + create_parser.add_argument("--head-sha", required=True) create_parser.add_argument("--title", required=True) - create_parser.add_argument("--body-file", type=Path) - - update_parser = subparsers.add_parser( - "update-draft", help="update title/body of an existing draft pull request" - ) - update_parser.add_argument("repo") - update_parser.add_argument("number", type=int) - update_parser.add_argument("--title") - update_parser.add_argument("--body-file", type=Path) + create_parser.add_argument("--body", default="") return parser.parse_args(argv) @@ -330,22 +422,14 @@ def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) - if args.operation == "create-draft": - data = { - "base": args.base, - "body": _body(args), - "head": args.head, - "title": _draft_title(args.title), - } - return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data - if args.number < 1: - raise PolicyError("pull-request number must be positive") - data = {} - if args.title is not None: - data["title"] = _draft_title(args.title) - if args.body_file is not None: - data["body"] = _body(args) - return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + _validate_sha(args.head_sha) + data = { + "base": _validate_ref(args.base, "base"), + "body": _validate_body(args.body), + "head": _validate_ref(args.head, "head"), + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data def _write_body(body: bytes) -> None: @@ -374,7 +458,6 @@ def main(argv: list[str] | None = None) -> int: "request_fields": sorted(data) if isinstance(data, dict) else [], - "requires_draft_preflight": operation == "update-draft", }, sort_keys=True, ) @@ -383,19 +466,17 @@ def main(argv: list[str] | None = None) -> int: token = read_token() if args.operation == "read": body = read(path, token=token) - elif args.operation == "create-draft": + else: assert isinstance(data, dict) body = create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), + head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), token=token, ) - else: - assert isinstance(data, dict) - body = update_draft(args.repo, args.number, data, token=token) _write_body(body) return 0 except urllib.error.HTTPError as exc: @@ -407,8 +488,6 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.buffer.write(b"\n") return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): - # Deliberately omit exception details. File contents, HTTP request - # objects, and provider errors can carry credential material. print( "Forgejo request rejected or unavailable; no credential was disclosed", file=sys.stderr, diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 4e73a3cf..32fd1ccf 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -1,6 +1,6 @@ --- name: manage-atlas-pull-requests -description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to update, merge, approve, close, delete, or force-push. --- # Manage Atlas pull requests @@ -12,47 +12,43 @@ put it in an argument or environment variable, or replace this client with ## Read repository or PR state -Pass an Atlas repository API path to the read operation: +Pass one allowed Atlas metadata API path to the read operation: ```sh /opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` -Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +The client exposes only repository metadata, PRs and PR evidence, branches, +commits, and commit status. Hooks, Actions secrets/variables, collaborators, +protections, reviews, merge endpoints, releases, and administration are +rejected even as reads. Responses are size-bounded. Treat returned state as evidence, not authorization to change it. ## Create a review handoff Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and -exact pushed commit. Never force-push. Write the PR description to a workspace -file so multiline Markdown is not shell-quoted, then validate without reading a -credential or using the network: +exact pushed commit. Never force-push. Pass that full 40-character pushed head +SHA explicitly. Keep the short body free of credentials and credential-like +text. Validate without reading a credential or using the network: ```sh /opt/coordinator/gitea_api.py --dry-run create-draft REPO \ - --base main --head hermes/TASK --title "Focused change" \ - --body-file /tmp/pr-body.md + --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ + --title "Focused change" --body "Tests and review evidence" ``` Run the same command without `--dry-run` only when the branch is review-ready. -The client always creates a draft in the same repository. Report its URL and -exact head commit to Brad. Leave it unmerged for human review. - -## Update an existing draft - -Use only a title, a body file, or both: - -```sh -/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ - --body-file /tmp/pr-body.md -``` - -The client first reads the PR from Forgejo and refuses the update unless it is -still a draft. It never changes head/base, review state, or merge state. +The client accepts success only when Forgejo returns the exact requested +repository, base/head refs, head SHA, title/body, canonical URLs, positive PR +number, and open/unmerged/draft state. Report its URL and exact head commit to +Brad. Leave it unmerged for human review. PR updates are intentionally absent; +publish a corrected tested commit and ask Brad how to proceed if metadata must +change. ## Stop at the authority boundary -Merge, approve, close, delete, branch mutation, comments, releases, repository -administration, and force-push are intentionally unavailable. Do not bypass the -client with a raw HTTP request. If one of those actions is needed, present the -tested commit and evidence to Brad and stop for human review. +Update, merge, approve, close, delete, branch mutation, comments, releases, +repository administration, and force-push are intentionally unavailable. Do +not bypass the client with a raw HTTP request. If one of those actions is +needed, present the tested commit and evidence to Brad and stop for human +review. diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 09f6b00c..025f3bf7 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -102,8 +102,12 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions - assert "Merge, approve, close, delete, comments" in instructions - assert "Leave every PR unmerged for Brad's review" in instructions + assert "Updates, merge,\napprove, close, delete, comments" in instructions + assert "Leave every PR\nunmerged for Brad's review" in instructions + assert "Use the Gitea pull-request merge endpoint" not in instructions + assert "reconcile the open PR" not in instructions + assert "use Flux reconciliation after" not in instructions + assert "PR publication ends at the verified open draft" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 658af189..91475bcf 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -2,18 +2,18 @@ from __future__ import annotations +import copy import importlib.util -import io import json import sys -import urllib.error +import urllib.request from pathlib import Path import pytest -import yaml ROOT = Path(__file__).parents[2] CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" def _load(): @@ -36,6 +36,27 @@ def _draft_payload(**updates): return payload +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + class Response: def __init__(self, body: object): self.body = body if isinstance(body, bytes) else json.dumps(body).encode() @@ -46,8 +67,32 @@ class Response: def __exit__(self, *_args): return False - def read(self): - return self.body + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] + + +def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): + client = _load() + source = urllib.request.Request( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + headers={"Authorization": "token redirect-sentinel"}, + ) + + with pytest.raises(client.PolicyError, match="redirects are not allowed") as exc: + client.RejectRedirectHandler().redirect_request( + source, + None, + 302, + "Found", + {}, + "https://evil.example/collect", + ) + + assert "redirect-sentinel" not in str(exc.value) + assert any( + isinstance(handler, client.RejectRedirectHandler) + for handler in client._SAFE_OPENER.handlers + ) @pytest.mark.parametrize( @@ -69,6 +114,71 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls?state=open&limit=20&page=1", + "/api/v1/repos/atlas/cassandra/pulls/7", + "/api/v1/repos/atlas/cassandra/pulls/7/commits?limit=20", + "/api/v1/repos/atlas/cassandra/pulls/7/files?page=1", + "/api/v1/repos/atlas/cassandra/branches", + "/api/v1/repos/atlas/cassandra/branches/main", + "/api/v1/repos/atlas/cassandra/commits?limit=10", + f"/api/v1/repos/atlas/cassandra/git/commits/{HEAD_SHA}", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/status", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/statuses?limit=10", + f"/api/v1/repos/atlas/cassandra/statuses/{HEAD_SHA}?page=1", + ], +) +def test_explicit_read_allowlist_accepts_only_engineering_metadata(path: str): + client = _load() + + request = client.build_request( + "GET", path, base_url=client.CANONICAL_BASE_URL, token="runtime" + ) + assert request.method == "GET" + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/repos/atlas/cassandra/actions/secrets", + "/api/v1/repos/atlas/cassandra/actions/variables", + "/api/v1/repos/atlas/cassandra/collaborators", + "/api/v1/repos/atlas/cassandra/branch_protections", + "/api/v1/repos/atlas/cassandra/keys", + "/api/v1/repos/atlas/cassandra/pulls/7/reviews", + "/api/v1/repos/atlas/cassandra/pulls/7/merge", + "/api/v1/repos/atlas/cassandra/pulls/7.diff", + "/api/v1/repos/atlas/cassandra/releases", + ], +) +def test_privileged_or_content_routes_are_denied_even_for_get(path: str): + client = _load() + + with pytest.raises(client.PolicyError, match="outside the metadata read allowlist"): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls?limit=51", + "/api/v1/repos/atlas/cassandra/pulls?state=merged", + "/api/v1/repos/atlas/cassandra/pulls?private=true", + "/api/v1/repos/atlas/cassandra/pulls?limit=1&limit=2", + "/api/v1/repos/atlas/cassandra?p=1", + ], +) +def test_read_query_is_bounded(path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request("GET", path, None) + + @pytest.mark.parametrize( ("method", "path", "data"), [ @@ -79,12 +189,11 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s "/api/v1/repos/atlas/cassandra/pulls/4/reviews", {"event": "APPROVED"}, ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), ], ) -def test_merge_approve_close_delete_and_other_mutations_are_rejected( +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( method: str, path: str, data: object ): client = _load() @@ -95,6 +204,36 @@ def test_merge_approve_close_delete_and_other_mutations_are_rejected( ) +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client.GIT_BIN == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + def test_create_forces_draft_title_and_same_repository_branch_names(): client = _load() assert ( @@ -103,7 +242,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) == "create-draft" ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): client.authorize_request( "POST", @@ -118,44 +256,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) -def test_update_checks_server_draft_state_before_patching(): - client = _load() - calls = [] - responses = iter( - [ - Response({"number": 7, "draft": True}), - Response({"number": 7, "draft": True}), - ] - ) - - def opener(request, timeout): - calls.append((request.method, request.full_url, request.data, timeout)) - return next(responses) - - result = client.update_draft( - "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener - ) - - assert json.loads(result) == {"number": 7, "draft": True} - assert [call[0] for call in calls] == ["GET", "PATCH"] - assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} - - -def test_update_rejects_non_draft_without_sending_patch(): - client = _load() - calls = [] - - def opener(request, timeout): - calls.append((request.method, timeout)) - return Response({"number": 7, "draft": False}) - - with pytest.raises(client.PolicyError, match="human review now owns"): - client.update_draft( - "cassandra", 7, {"body": "new"}, token="runtime", opener=opener - ) - assert calls == [("GET", 30)] - - def test_runtime_token_is_only_an_authorization_header(): client = _load() request = client.build_request( @@ -171,41 +271,128 @@ def test_runtime_token_is_only_an_authorization_header(): assert request.get_header("Authorization") == "token do-not-leak" -def test_create_uses_live_gitea_schema_and_verifies_draft_response(): +@pytest.mark.parametrize( + "body", + [ + "pass" + "word=not-a-real-credential", + "Authorization: " + "Bearer not-a-real-credential-value", + "token: " + "ghp_" + "notarealcredentialvalue123456", + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +def test_body_rejects_common_credential_shapes(body: str): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client._validate_body(body) + + +def test_create_rejects_exact_runtime_token_before_network(): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="accidental runtime-sentinel value", + token="runtime-sentinel", + opener=opener, + ) + assert called is False + + +def test_create_verifies_every_server_postcondition(): client = _load() calls = [] def opener(request, timeout): - calls.append((json.loads(request.data), timeout)) - return Response({"number": 3, "draft": True}) + calls.append((request, timeout)) + return Response(_draft_response()) result = client.create_draft( "cassandra", base="main", head="hermes/fix", + head_sha=HEAD_SHA, title="Focused fix", - body="Evidence", + body="Review evidence", token="runtime", opener=opener, ) - assert json.loads(result)["draft"] is True - assert calls[0][0]["title"] == "WIP: Focused fix" - assert "draft" not in calls[0][0] + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 -def test_create_fails_closed_when_server_does_not_confirm_draft(): +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): client = _load() - with pytest.raises(client.PolicyError, match="did not confirm draft"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - title="Focused fix", - body="Evidence", + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", token="runtime", - opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), ) @@ -216,107 +403,3 @@ def test_output_redaction_covers_exact_token_and_authorization_header(): assert b"do-not-leak" not in redacted assert redacted.count(b"[REDACTED]") >= 1 - - -def test_dry_run_does_not_read_token_or_use_network( - tmp_path: Path, monkeypatch, capsys -): - client = _load() - body = tmp_path / "body.md" - body.write_text("Evidence only\n", encoding="utf-8") - monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) - monkeypatch.setattr( - client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") - ) - - assert ( - client.main( - [ - "--dry-run", - "create-draft", - "cassandra", - "--base", - "main", - "--head", - "hermes/fix", - "--title", - "Repair", - "--body-file", - str(body), - ] - ) - == 0 - ) - output = json.loads(capsys.readouterr().out) - assert output["operation"] == "create-draft" - assert output["owner"] == "atlas" - assert "Evidence only" not in json.dumps(output) - - -def test_http_error_path_redacts_token(monkeypatch, capsys): - client = _load() - monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") - - def fail(*_args, **_kwargs): - raise urllib.error.HTTPError( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", - 403, - "forbidden", - {}, - io.BytesIO(b"Authorization: token do-not-leak"), - ) - - monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) - - assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 - captured = capsys.readouterr() - assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err - - -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): - client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source - assert "GITEA_TOKEN" not in client_source - - deployment = yaml.safe_load( - (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") - ) - template = deployment["spec"]["template"] - annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) - runtime = next( - volume - for volume in template["spec"]["volumes"] - if volume["name"] == "runtime-access" - ) - assert runtime["emptyDir"]["medium"] == "Memory" - - expected = {"hermes", "terminal", "cli-lane-runner"} - mounted = { - container["name"] - for container in template["spec"]["containers"] - if any( - mount["name"] == "atlas-pr-skill" - and mount["mountPath"] - == "/opt/data/workspace/skills/manage-atlas-pull-requests" - and mount.get("readOnly") is True - for mount in container.get("volumeMounts", []) - ) - } - assert mounted == expected - - kustomization = yaml.safe_load( - (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") - ) - generator = next( - item - for item in kustomization["configMapGenerator"] - if item["name"] == "hermes-atlas-pr-skill" - ) - assert generator["files"] == [ - "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", - "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", - ] diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py new file mode 100644 index 00000000..d3b47eef --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -0,0 +1,147 @@ +"""Runtime and Flux integration contracts for the safe Atlas PR client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" + + +def _load(): + spec = importlib.util.spec_from_file_location( + "safe_gitea_api_integration", CLIENT_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_dry_run_has_no_file_input_and_uses_no_token_or_network(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client, "_safe_urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body", + "Evidence only", + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + with pytest.raises(SystemExit): + client.parse_args( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body-file", + "/runtime-access/gitea-token", + ] + ) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] From 2c1d97e551e5b61a3aa1e821e412ef4948bbcc8d Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:21:04 -0300 Subject: [PATCH 10/46] hermes: match canonical Forgejo PR URL --- services/hermes/scripts/gitea_api.py | 3 +-- testing/tests/test_hermes_gitea_pr_client.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 057a79a1..b4a98eca 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -337,13 +337,12 @@ def _require_create_response( raise PolicyError("Forgejo omitted a valid pull-request number") full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" - expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" checks = ( document.get("state") == "open", document.get("draft") is True, document.get("merged") is False, document.get("html_url") == expected_html, - document.get("url") == expected_api, + document.get("url") == expected_html, document.get("title") == title, document.get("body") == body, _nested(document, "base", "ref") == base, diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 91475bcf..447f3067 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -43,7 +43,7 @@ def _draft_response(**updates): "draft": True, "merged": False, "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", "title": "WIP: Focused fix", "body": "Review evidence", "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, From 8c6e3acdacb223f68b1716f0a74bd543e18890e6 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:41:14 -0300 Subject: [PATCH 11/46] hermes: isolate Atlas SCM write authority --- .../kustomization.yaml | 19 + .../hermes-observer-rbac/kustomization.yaml | 16 + .../hermes-scm-broker-code/kustomization.yaml | 19 + .../hermes-scm-broker/kustomization.yaml | 27 + .../hermes-scm-namespace/kustomization.yaml | 16 + .../applications/hermes/kustomization.yaml | 2 + .../applications/kustomization.yaml | 5 + scripts/tests/test_dashboards_render_atlas.py | 98 - ...test_dashboards_render_atlas_drilldowns.py | 127 ++ .../gitea/atlas-identity-bootstrap-job.yaml | 20 +- services/gitea/kustomization.yaml | 1 + .../scripts/gitea_atlas_identity_ensure.sh | 45 + .../scripts/gitea_branch_protection_check.py | 207 ++ .../kustomization.yaml | 5 + .../rolebindings.yaml | 204 ++ .../hermes-observer-rbac/kustomization.yaml | 5 + services/hermes-observer-rbac/rbac.yaml | 112 + services/hermes-scm-broker/deployment.yaml | 98 + services/hermes-scm-broker/kustomization.yaml | 11 + services/hermes-scm-broker/networkpolicy.yaml | 46 + services/hermes-scm-broker/service.yaml | 13 + .../hermes-scm-namespace/kustomization.yaml | 5 + services/hermes-scm-namespace/namespace.yaml | 14 + services/hermes/agent-configmap.yaml | 74 +- services/hermes/agent-deployment.yaml | 20 +- services/hermes/agent-kubeconfig.yaml | 4 +- services/hermes/agent-rbac.yaml | 16 - services/hermes/kustomization.yaml | 12 +- services/hermes/networkpolicy.yaml | 62 +- services/hermes/node-ssh-access.yaml | 77 +- services/hermes/scm-common/kustomization.yaml | 14 + .../{ => scm-common}/scripts/gitea_api.py | 284 ++- .../scm-common/scripts/gitea_api_policy.py | 490 +++++ .../hermes/scm-common/scripts/scm_broker.py | 496 +++++ .../scm-common/scripts/scm_broker_client.py | 95 + .../scm-common/scripts/scm_broker_io.py | 72 + .../scm-common/scripts/scm_broker_server.py | 118 ++ services/hermes/scripts/gitea_askpass.sh | 19 - services/hermes/scripts/hermes_coordinator.py | 32 +- services/hermes/scripts/node_account_audit.py | 178 ++ .../hermes/scripts/node_account_hardening.py | 496 +++++ services/hermes/scripts/node_account_io.py | 205 ++ .../hermes/scripts/stage_runtime_access.py | 19 +- .../manage-atlas-pull-requests/SKILL.md | 24 +- .../vault/hermes-auth-role-bootstrap-job.yaml | 2 +- .../vault/scripts/vault_k8s_auth_configure.sh | 4 +- testing/quality_contract.json | 57 + testing/quality_coverage.py | 61 +- testing/tests/test_hermes_chat_images.py | 282 +++ .../tests/test_hermes_chat_local_runtime.py | 172 ++ .../tests/test_hermes_chat_provider_auth.py | 476 +++++ testing/tests/test_hermes_chat_quality.py | 1812 +---------------- .../tests/test_hermes_chat_routing_runtime.py | 486 +++++ .../test_hermes_chat_session_continuity.py | 478 +++++ testing/tests/test_hermes_chat_support.py | 38 + testing/tests/test_hermes_cli_lanes.py | 1591 +-------------- testing/tests/test_hermes_cli_lanes_access.py | 378 ++++ .../test_hermes_cli_lanes_configuration.py | 444 ++++ testing/tests/test_hermes_cli_lanes_kanban.py | 392 ++++ .../tests/test_hermes_cli_lanes_support.py | 79 + .../tests/test_hermes_cli_lanes_toolchain.py | 386 ++++ testing/tests/test_hermes_coordinator.py | 212 +- .../test_hermes_coordinator_cassandra.py | 194 ++ .../tests/test_hermes_coordinator_coverage.py | 267 +++ .../tests/test_hermes_coordinator_support.py | 35 + .../test_hermes_gitea_branch_coverage.py | 183 ++ .../test_hermes_gitea_internal_coverage.py | 375 ++++ testing/tests/test_hermes_gitea_pr_client.py | 398 ++-- .../tests/test_hermes_gitea_pr_creation.py | 203 ++ testing/tests/test_hermes_gitea_pr_dlp.py | 317 +++ .../tests/test_hermes_gitea_pr_integration.py | 41 +- .../test_hermes_gitea_pr_postconditions.py | 199 ++ testing/tests/test_hermes_gitea_support.py | 75 + .../test_hermes_node_account_hardening.py | 252 +++ testing/tests/test_hermes_node_account_io.py | 222 ++ ...est_hermes_node_account_privilege_audit.py | 196 ++ .../tests/test_hermes_node_account_support.py | 90 + .../tests/test_hermes_node_acl_coverage.py | 206 ++ .../tests/test_hermes_node_audit_coverage.py | 221 ++ .../test_hermes_node_hardening_coverage.py | 211 ++ testing/tests/test_hermes_node_io_coverage.py | 189 ++ testing/tests/test_hermes_runtime_access.py | 24 +- .../test_hermes_runtime_stage_coverage.py | 137 ++ testing/tests/test_hermes_scm_broker.py | 146 ++ .../test_hermes_scm_broker_client_coverage.py | 157 ++ ...test_hermes_scm_broker_handler_coverage.py | 265 +++ ...est_hermes_scm_broker_internal_coverage.py | 263 +++ .../tests/test_hermes_scm_broker_policy.py | 321 +++ .../tests/test_hermes_scm_broker_streaming.py | 237 +++ .../tests/test_hermes_scm_broker_support.py | 55 + .../tests/test_hermes_scm_server_coverage.py | 180 ++ .../tests/test_quality_coverage_helpers.py | 87 +- 92 files changed, 12731 insertions(+), 4287 deletions(-) create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml create mode 100644 scripts/tests/test_dashboards_render_atlas_drilldowns.py create mode 100644 services/gitea/scripts/gitea_branch_protection_check.py create mode 100644 services/hermes-observer-bindings/kustomization.yaml create mode 100644 services/hermes-observer-bindings/rolebindings.yaml create mode 100644 services/hermes-observer-rbac/kustomization.yaml create mode 100644 services/hermes-observer-rbac/rbac.yaml create mode 100644 services/hermes-scm-broker/deployment.yaml create mode 100644 services/hermes-scm-broker/kustomization.yaml create mode 100644 services/hermes-scm-broker/networkpolicy.yaml create mode 100644 services/hermes-scm-broker/service.yaml create mode 100644 services/hermes-scm-namespace/kustomization.yaml create mode 100644 services/hermes-scm-namespace/namespace.yaml delete mode 100644 services/hermes/agent-rbac.yaml create mode 100644 services/hermes/scm-common/kustomization.yaml rename services/hermes/{ => scm-common}/scripts/gitea_api.py (65%) mode change 100755 => 100644 create mode 100644 services/hermes/scm-common/scripts/gitea_api_policy.py create mode 100644 services/hermes/scm-common/scripts/scm_broker.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_client.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_io.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_server.py delete mode 100755 services/hermes/scripts/gitea_askpass.sh create mode 100644 services/hermes/scripts/node_account_audit.py create mode 100644 services/hermes/scripts/node_account_hardening.py create mode 100644 services/hermes/scripts/node_account_io.py create mode 100644 testing/tests/test_hermes_chat_images.py create mode 100644 testing/tests/test_hermes_chat_local_runtime.py create mode 100644 testing/tests/test_hermes_chat_provider_auth.py create mode 100644 testing/tests/test_hermes_chat_routing_runtime.py create mode 100644 testing/tests/test_hermes_chat_session_continuity.py create mode 100644 testing/tests/test_hermes_chat_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_access.py create mode 100644 testing/tests/test_hermes_cli_lanes_configuration.py create mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_cli_lanes_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_toolchain.py create mode 100644 testing/tests/test_hermes_coordinator_cassandra.py create mode 100644 testing/tests/test_hermes_coordinator_coverage.py create mode 100644 testing/tests/test_hermes_coordinator_support.py create mode 100644 testing/tests/test_hermes_gitea_branch_coverage.py create mode 100644 testing/tests/test_hermes_gitea_internal_coverage.py create mode 100644 testing/tests/test_hermes_gitea_pr_creation.py create mode 100644 testing/tests/test_hermes_gitea_pr_dlp.py create mode 100644 testing/tests/test_hermes_gitea_pr_postconditions.py create mode 100644 testing/tests/test_hermes_gitea_support.py create mode 100644 testing/tests/test_hermes_node_account_hardening.py create mode 100644 testing/tests/test_hermes_node_account_io.py create mode 100644 testing/tests/test_hermes_node_account_privilege_audit.py create mode 100644 testing/tests/test_hermes_node_account_support.py create mode 100644 testing/tests/test_hermes_node_acl_coverage.py create mode 100644 testing/tests/test_hermes_node_audit_coverage.py create mode 100644 testing/tests/test_hermes_node_hardening_coverage.py create mode 100644 testing/tests/test_hermes_node_io_coverage.py create mode 100644 testing/tests/test_hermes_runtime_stage_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker.py create mode 100644 testing/tests/test_hermes_scm_broker_client_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_handler_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_internal_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_policy.py create mode 100644 testing/tests/test_hermes_scm_broker_streaming.py create mode 100644 testing/tests/test_hermes_scm_broker_support.py create mode 100644 testing/tests/test_hermes_scm_server_coverage.py diff --git a/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..b64913d5 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-bindings + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-bindings + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..c2749739 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-rbac + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-rbac + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml new file mode 100644 index 00000000..20ae82e2 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker-code + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes/scm-common + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes-scm-namespace diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..70b3989e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,27 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-broker + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 10m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-scm-broker + namespace: hermes-scm + dependsOn: + - name: vault + - name: gitea + - name: hermes-scm-namespace + - name: hermes-scm-broker-code diff --git a/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..9f066709 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-namespace + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-namespace + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..df7a5560 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,5 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: hermes-scm-broker + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..39a859d0 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -27,7 +27,12 @@ resources: - jenkins/kustomization.yaml - ai-llm/kustomization.yaml - openclaw/kustomization.yaml + - hermes-scm-namespace/kustomization.yaml + - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes-observer-rbac/kustomization.yaml + - hermes-observer-bindings/kustomization.yaml + - hermes-scm-broker/kustomization.yaml - hermes-chat/kustomization.yaml - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index bba16cdf..a522f849 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -416,101 +416,3 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): assert {"color": "dark-yellow", "value": 93} in threshold_steps assert {"color": "dark-blue", "value": 100} in threshold_steps - - -def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): - mod = load_module() - dashboard = mod.build_jobs_dashboard() - panels = dashboard["panels"] - rows = [panel for panel in panels if panel["type"] == "row"] - visible_query_panels = [panel for panel in panels if panel["type"] != "row"] - nested_panels_by_title = { - child["title"]: child - for row in rows - for child in row.get("panels", []) - } - - assert len(panels) == 18 - assert len(visible_query_panels) == 12 - assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 - assert all( - panel["title"] != "Coverage Gap to 95% by Suite" - for panel in visible_query_panels - ) - assert [row["title"] for row in rows] == [ - "CI Runs And Test Result History", - "Check Failure Rates By Suite", - "Check Healthy Rates By Suite", - "Test Drilldowns And Problem Tests", - "Telemetry Completeness And Branches", - "SonarQube Project Health", - ] - assert all(row["collapsed"] for row in rows) - - assert "Coverage Failure Rate" in nested_panels_by_title - assert "Supply Chain Healthy Rate" in nested_panels_by_title - assert "Test Category Health History" in nested_panels_by_title - assert "Selected Test Pass Rate History" in nested_panels_by_title - assert "Coverage Metrics Present by Suite" in nested_panels_by_title - assert "SonarQube API Up" in nested_panels_by_title - - failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] - assert failure_rate_panel["type"] == "state-timeline" - assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 - assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue" - assert "increase(" not in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"] - assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"] - assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] - assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"] - assert "0 *" in failure_rate_panel["targets"][0]["expr"] - assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] - - pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] - assert pass_rate_panel["type"] == "state-timeline" - assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"] - assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"] - - pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] - assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" - assert all( - "platform_quality:test_case_status:count_1h" in target["expr"] - for target in pass_fail_panel["targets"] - ) - - problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"] - assert problematic_panel["type"] == "state-timeline" - assert problematic_panel["gridPos"]["w"] == 24 - assert 'test!=""' in problematic_panel["targets"][0]["expr"] - assert "vector(0)" not in problematic_panel["targets"][0]["expr"] - assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ - {"color": "dark-blue", "value": None}, - {"color": "dark-green", "value": 2}, - {"color": "dark-yellow", "value": 3}, - {"color": "dark-orange", "value": 5}, - {"color": "dark-red", "value": 8}, - ] - assert "rolling 24h failure count" in problematic_panel["description"] - assert "at least two recent failures" in problematic_panel["description"] - - sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] - sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] - assert sonar_mix_panel["gridPos"]["w"] == 4 - assert sonar_health_panel["gridPos"]["w"] == 8 - assert sonar_health_panel["type"] == "state-timeline" - assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] - assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] - - branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] - recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] - assert branch_panel["gridPos"]["x"] == 12 - assert recent_branch_panel["gridPos"]["x"] == 18 - assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] - assert "[7d:1h]" in branch_panel["targets"][0]["expr"] - assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert "unless on(suite)" in branch_panel["targets"][0]["expr"] - assert "> bool 0" in branch_panel["targets"][0]["expr"] - assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py new file mode 100644 index 00000000..08871b71 --- /dev/null +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -0,0 +1,127 @@ +"""Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" + +from scripts.tests.test_dashboards_render_atlas import load_module + + +def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): + mod = load_module() + dashboard = mod.build_jobs_dashboard() + panels = dashboard["panels"] + rows = [panel for panel in panels if panel["type"] == "row"] + visible_query_panels = [panel for panel in panels if panel["type"] != "row"] + nested_panels_by_title = { + child["title"]: child for row in rows for child in row.get("panels", []) + } + + assert len(panels) == 18 + assert len(visible_query_panels) == 12 + assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 + assert all( + panel["title"] != "Coverage Gap to 95% by Suite" + for panel in visible_query_panels + ) + assert [row["title"] for row in rows] == [ + "CI Runs And Test Result History", + "Check Failure Rates By Suite", + "Check Healthy Rates By Suite", + "Test Drilldowns And Problem Tests", + "Telemetry Completeness And Branches", + "SonarQube Project Health", + ] + assert all(row["collapsed"] for row in rows) + + assert "Coverage Failure Rate" in nested_panels_by_title + assert "Supply Chain Healthy Rate" in nested_panels_by_title + assert "Test Category Health History" in nested_panels_by_title + assert "Selected Test Pass Rate History" in nested_panels_by_title + assert "Coverage Metrics Present by Suite" in nested_panels_by_title + assert "SonarQube API Up" in nested_panels_by_title + + failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] + assert failure_rate_panel["type"] == "state-timeline" + assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 + assert ( + failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] + == "dark-blue" + ) + assert "increase(" not in failure_rate_panel["targets"][0]["expr"] + assert ( + "platform_quality:check_failed_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_seen_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_status:present_1h" + not in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + ".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"] + ) + assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] + assert "label_replace" not in failure_rate_panel["targets"][0]["expr"] + assert "0 *" in failure_rate_panel["targets"][0]["expr"] + assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] + + pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] + assert pass_rate_panel["type"] == "state-timeline" + assert ( + "platform_quality:test_case_pass_rate:percent_1h" + in pass_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality_gate_test_case_result" + not in pass_rate_panel["targets"][0]["expr"] + ) + + pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] + assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" + assert all( + "platform_quality:test_case_status:count_1h" in target["expr"] + for target in pass_fail_panel["targets"] + ) + + problematic_panel = nested_panels_by_title[ + "Problematic Tests Over Time (Top failures)" + ] + assert problematic_panel["type"] == "state-timeline" + assert problematic_panel["gridPos"]["w"] == 24 + assert 'test!=""' in problematic_panel["targets"][0]["expr"] + assert "vector(0)" not in problematic_panel["targets"][0]["expr"] + assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ + {"color": "dark-blue", "value": None}, + {"color": "dark-green", "value": 2}, + {"color": "dark-yellow", "value": 3}, + {"color": "dark-orange", "value": 5}, + {"color": "dark-red", "value": 8}, + ] + assert "rolling 24h failure count" in problematic_panel["description"] + assert "at least two recent failures" in problematic_panel["description"] + + sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] + sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] + assert sonar_mix_panel["gridPos"]["w"] == 4 + assert sonar_health_panel["gridPos"]["w"] == 8 + assert sonar_health_panel["type"] == "state-timeline" + assert ( + "platform_quality:sonar_gate_health_percent:latest_1h" + in sonar_health_panel["targets"][0]["expr"] + ) + assert ( + "sonarqube_project_quality_gate_pass" + not in sonar_health_panel["targets"][0]["expr"] + ) + + branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] + recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] + assert branch_panel["gridPos"]["x"] == 12 + assert recent_branch_panel["gridPos"]["x"] == 18 + assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in branch_panel["targets"][0]["expr"] + assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert "unless on(suite)" in branch_panel["targets"][0]["expr"] + assert "> bool 0" in branch_panel["targets"][0]["expr"] + assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index c7e2071f..bacecd01 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-3 + name: gitea-atlas-identity-bootstrap-5 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -34,6 +34,20 @@ spec: nodeSelector: node-role.kubernetes.io/worker: "true" hardware: rpi5 + initContainers: + - name: python-runtime + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + command: [/bin/sh, -ec] + args: ["cp -R /usr/local/. /python/"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - {name: python-runtime, mountPath: /python} containers: - name: bootstrap image: gitea/gitea:1.23 @@ -80,6 +94,7 @@ spec: - name: bootstrap mountPath: /opt/bootstrap readOnly: true + - {name: python-runtime, mountPath: /opt/python, readOnly: true} volumes: - name: data persistentVolumeClaim: @@ -88,3 +103,6 @@ spec: configMap: name: gitea-atlas-identity-bootstrap defaultMode: 0555 + - name: python-runtime + emptyDir: + sizeLimit: 128Mi diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index 8c473a38..6511418a 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -15,6 +15,7 @@ configMapGenerator: - name: gitea-atlas-identity-bootstrap files: - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh + - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py generatorOptions: disableNameSuffixHash: true diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index f98f08f5..6d5dbd90 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}" managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' +protected_reviewer=bstein die() { echo "Gitea Atlas identity bootstrap failed: $*" >&2 @@ -272,6 +273,46 @@ transfer_managed_repositories() { trap - EXIT HUP INT TERM } +ensure_default_branch_protection() { + repository=$1 + repo_file=$(mktemp) + protections_file=$(mktemp) + response_file=$(mktemp) + + status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}") + expect_status "${status}" 200 "read ${repository} metadata for branch protection" + default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}") + case "${default_branch}" in + main|master) ;; + *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;; + esac + + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "list ${repository} branch protections" + if ! protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ); then + die "${repository} ${default_branch} protection differs from the human-review policy" + fi + + if [ "${protection_state}" = ABSENT ]; then + payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}" + status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}") + expect_status "${status}" 201 "protect ${repository} ${default_branch}" + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "read back ${repository} branch protections" + protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ) || die "${repository} ${default_branch} protection differs from the human-review policy" + fi + [ "${protection_state}" = PRESENT ] || \ + die "${repository} ${default_branch} protection did not become effective" + + rm -f "${repo_file}" "${protections_file}" "${response_file}" +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors" transfer_managed_repositories +for repository in ${managed_repositories}; do + ensure_default_branch_protection "${repository}" +done + vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py new file mode 100644 index 00000000..e3f112c9 --- /dev/null +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Validate the first effective Gitea protection for an Atlas primary branch.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import stat +from pathlib import Path + +MAX_INPUT = 1024 * 1024 +MAX_RULES = 100 +MAX_RULE_NAME = 255 +SPECIAL = frozenset("*?\\[]{}") + + +class PolicyError(RuntimeError): + """The protection response is ambiguous or violates human review policy.""" + + +def _read_bounded(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT: + raise PolicyError("branch protection response exceeds the safe limit") + value = os.read(descriptor, MAX_INPUT + 1) + finally: + os.close(descriptor) + if len(value) != metadata.st_size: + raise PolicyError("branch protection response changed while reading") + return value + + +def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]: + """Compile the gobwas/glob syntax Gitea v1.23.8 uses.""" + pieces: list[str] = [] + while position < len(pattern): + character = pattern[position] + if character in terminators: + break + if character == "\\": + position += 1 + if position >= len(pattern): + raise ValueError("trailing escape") + pieces.append(re.escape(pattern[position])) + elif character == "*": + if position + 1 < len(pattern) and pattern[position + 1] == "*": + pieces.append(".*") + position += 1 + else: + pieces.append("[^/]*") + elif character == "?": + pieces.append("[^/]") + elif character == "[": + end = pattern.find("]", position + 1) + if end < 0: + raise ValueError("unterminated range") + value = pattern[position + 1 : end] + if not value: + raise ValueError("empty range") + negate = value.startswith("!") + value = value[1:] if negate else value + if not value or "[" in value or "\\" in value: + raise ValueError("invalid range") + if "-" in value: + if value.count("-") != 1 or value.startswith("-") or value.endswith("-"): + raise ValueError("invalid range") + low, high = value.split("-") + if len(low) != 1 or len(high) != 1 or ord(high) < ord(low): + raise ValueError("invalid range") + content = re.escape(low) + "-" + re.escape(high) + else: + content = re.escape(value) + pieces.append("[" + ("^" if negate else "") + content + "]") + position = end + elif character == "{": + alternatives: list[str] = [] + position += 1 + while True: + alternative, position = _glob_regex(pattern, position, ",}") + alternatives.append(alternative) + if position >= len(pattern): + raise ValueError("unterminated alternatives") + if pattern[position] == "}": + break + position += 1 + if len(alternatives) < 2 or any(not item for item in alternatives): + raise ValueError("invalid alternatives") + pieces.append("(?:" + "|".join(alternatives) + ")") + elif character in "]}": + # A closing delimiter outside its grammar context is literal in + # gobwas/glob's lexer. + pieces.append(re.escape(character)) + else: + pieces.append(re.escape(character)) + position += 1 + return "".join(pieces), position + + +def _is_plain(pattern: str) -> bool: + return not any(character in SPECIAL for character in pattern) + + +def _matches(pattern: str, branch: str) -> bool: + if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: + raise PolicyError("branch protection rule name is invalid") + if _is_plain(pattern): + return pattern.casefold() == branch.casefold() + try: + expression, position = _glob_regex(pattern) + if position != len(pattern): + raise ValueError("incomplete glob") + except (re.error, ValueError): + # Gitea quotes an invalid special pattern and matches it literally. + # A literal containing a special byte cannot equal main or master. + return False + return re.fullmatch(expression, branch) is not None + + +def _created(value: object) -> datetime.datetime: + if not isinstance(value, str) or len(value) > 64: + raise PolicyError("branch protection creation time is invalid") + try: + result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PolicyError("branch protection creation time is invalid") from exc + if result.tzinfo is None: + raise PolicyError("branch protection creation time is invalid") + return result + + +def _required(reviewer: str) -> dict[str, object]: + return { + "enable_push": True, + "enable_push_whitelist": True, + "push_whitelist_usernames": [reviewer], + "push_whitelist_deploy_keys": False, + "enable_force_push": False, + "enable_merge_whitelist": True, + "merge_whitelist_usernames": [reviewer], + "enable_approvals_whitelist": True, + "approvals_whitelist_username": [reviewer], + "required_approvals": 1, + "block_on_rejected_reviews": True, + "block_on_outdated_branch": True, + "dismiss_stale_approvals": True, + "block_admin_merge_override": True, + } + + +def evaluate(value: bytes, branch: str, reviewer: str) -> str: + """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift.""" + if len(value) > MAX_INPUT or branch not in {"main", "master"}: + raise PolicyError("branch protection input is invalid") + data = json.loads(value) + if not isinstance(data, list) or len(data) > MAX_RULES: + raise PolicyError("branch protection response is invalid") + ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = [] + for index, raw in enumerate(data): + if not isinstance(raw, dict): + raise PolicyError("branch protection entry is invalid") + priority = raw.get("priority") + rule_name = raw.get("rule_name") + if ( + not isinstance(priority, int) + or isinstance(priority, bool) + or not 1 <= priority <= 1_000_000 + or not isinstance(rule_name, str) + ): + raise PolicyError("branch protection priority is ambiguous") + created = _created(raw.get("created_at")) + if _matches(rule_name, branch): + # Gitea v1.23.8 sorts by priority, then puts plain names before + # globs, then uses creation time. The API exposes every component. + ordered.append((priority, not _is_plain(rule_name), created, index, raw)) + if not ordered: + return "ABSENT" + effective = min(ordered)[4] + for key, expected in _required(reviewer).items(): + if effective.get(key) != expected: + raise PolicyError( + f"effective {branch} protection differs from human-review policy" + ) + return "PRESENT" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("branch", choices=("main", "master")) + parser.add_argument("reviewer") + args = parser.parse_args() + try: + value = _read_bounded(args.response) + print(evaluate(value, args.branch, args.reviewer)) + except (OSError, json.JSONDecodeError, PolicyError) as exc: + print(f"branch protection check failed: {exc}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes-observer-bindings/kustomization.yaml b/services/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..4dbf36b5 --- /dev/null +++ b/services/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rolebindings.yaml diff --git a/services/hermes-observer-bindings/rolebindings.yaml b/services/hermes-observer-bindings/rolebindings.yaml new file mode 100644 index 00000000..403cd39a --- /dev/null +++ b/services/hermes-observer-bindings/rolebindings.yaml @@ -0,0 +1,204 @@ +# services/hermes-observer-bindings/rolebindings.yaml +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: ai} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: bstein-dev-home} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cassandra} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cert-manager} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: climate} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: comms} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: crypto} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: default} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: finance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: flux-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: game-stream} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: gitea} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: harbor} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: health} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-chat} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jellyfin} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jenkins} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-node-lease} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-public} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: logging} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: longhorn-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: mailu-mailserver} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: maintenance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: metallb-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: monitoring} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: nextcloud} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: openclaw} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: outline} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: planka} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: postgres} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: quality} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sso} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sui-metrics} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: traefik} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vault} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vaultwarden} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: veles} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} diff --git a/services/hermes-observer-rbac/kustomization.yaml b/services/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..cff6f4b9 --- /dev/null +++ b/services/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/services/hermes-observer-rbac/rbac.yaml b/services/hermes-observer-rbac/rbac.yaml new file mode 100644 index 00000000..737ac4dd --- /dev/null +++ b/services/hermes-observer-rbac/rbac.yaml @@ -0,0 +1,112 @@ +# services/hermes-observer-rbac/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - persistentvolumes + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-namespaced-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - configmaps + - endpoints + - events + - persistentvolumeclaims + - pods + - pods/log + - replicationcontrollers + - resourcequotas + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "networkpolicies"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - source.toolkit.fluxcd.io + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["longhorn.io"] + resources: + - backingimages + - engines + - instancemanagers + - nodes + - replicas + - settings + - volumes + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +subjects: + - kind: ServiceAccount + name: hermes-agent + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-agent-cluster-observer-v2 diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml new file mode 100644 index 00000000..649ada1d --- /dev/null +++ b/services/hermes-scm-broker/deployment.yaml @@ -0,0 +1,98 @@ +# services/hermes-scm-broker/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-scm-broker + namespace: hermes-scm + labels: + app: hermes-scm-broker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-scm-broker + template: + metadata: + labels: + app: hermes-scm-broker + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-scm-broker + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} + {{ .Data.data.token }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 10m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 50m + vault.hashicorp.com/agent-limits-mem: 64Mi + spec: + serviceAccountName: hermes-scm-broker + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + containers: + - name: broker + image: registry.bstein.dev/bstein/hermes-agent + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/broker/scm_broker.py + ports: + - {name: http, containerPort: 9081, protocol: TCP} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + volumeMounts: + - {name: broker-code, mountPath: /opt/broker, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 768Mi} + volumes: + - name: broker-code + configMap: + name: hermes-scm-boundary + defaultMode: 0555 + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/services/hermes-scm-broker/kustomization.yaml b/services/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..5dac3620 --- /dev/null +++ b/services/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,11 @@ +# services/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-scm +resources: + - service.yaml + - deployment.yaml + - networkpolicy.yaml +images: + - name: registry.bstein.dev/bstein/hermes-agent + digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml new file mode 100644 index 00000000..8187e333 --- /dev/null +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -0,0 +1,46 @@ +# services/hermes-scm-broker/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-scm-broker-isolation + namespace: hermes-scm +spec: + podSelector: + matchLabels: + app: hermes-scm-broker + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-agent + ports: + - {protocol: TCP, port: 9081} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 192.168.22.9/32 + ports: + - {protocol: TCP, port: 443} diff --git a/services/hermes-scm-broker/service.yaml b/services/hermes-scm-broker/service.yaml new file mode 100644 index 00000000..0b08c7f6 --- /dev/null +++ b/services/hermes-scm-broker/service.yaml @@ -0,0 +1,13 @@ +# services/hermes-scm-broker/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-scm-broker + namespace: hermes-scm +spec: + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http diff --git a/services/hermes-scm-namespace/kustomization.yaml b/services/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..a9e89c7b --- /dev/null +++ b/services/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/services/hermes-scm-namespace/namespace.yaml b/services/hermes-scm-namespace/namespace.yaml new file mode 100644 index 00000000..542f9602 --- /dev/null +++ b/services/hermes-scm-namespace/namespace.yaml @@ -0,0 +1,14 @@ +# services/hermes-scm-namespace/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-scm + labels: + app.kubernetes.io/name: hermes-scm-broker +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-scm-broker + namespace: hermes-scm +automountServiceAccountToken: true diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 782876cf..b53b2c92 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -189,9 +189,9 @@ data: not replace these coordinator-wide rules. Never call `kanban_show` without a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not need a synthetic Kanban lookup, and must load a skill only when its workflow - materially applies. Atlas HTTPS Git authentication is already supplied by - the runtime-only `GIT_ASKPASS`; use it without reading or exposing the - credential. Coordinator guidance lives at + materially applies. Atlas Git access is supplied by the isolated SCM + broker; no repository token is present in this pod. Use only configured + broker remotes and clients. Coordinator guidance lives at `/opt/data/workspace/AGENTS.md` when more detail is needed. The Jetson classifier is mandatory for AUTO selection. Switchyard may use @@ -295,21 +295,21 @@ data: objective's difficulty warrants it; otherwise synthesize at the original effort. - This owner-only pod has cluster-admin access across Atlas, including logs, - Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation. - Prefer the titan-iac Git/Flux workflow for every durable cluster change; - direct operations are available for explicit operator requests, incident - recovery, and verification, and must be followed by a matching source-of- - truth change when they alter desired state. Never expose credentials in - chat or logs. Triage belongs at triage.hermes.bstein.dev. + This owner-only pod has scoped read-only Kubernetes diagnostics across + Atlas, including resource status, events, logs, and Flux/Helm evidence. It + cannot read Secrets, exec or attach to pods, create service-account tokens, + mutate workloads or RBAC, or reconcile Flux. Put every durable cluster + change on a reviewed titan-iac branch. Never expose credentials in chat or + logs. Triage belongs at triage.hermes.bstein.dev. ## Atlas engineering access The Atlas organization has private visibility. Repository visibility is preserved per project and may be public or private; do not infer a repository's visibility from the organization setting. Repositories are - canonical at `https://scm.bstein.dev/atlas/.git`. HTTPS Git - authentication is already supplied through `GIT_ASKPASS`. Verify the remote + canonical at `https://scm.bstein.dev/atlas/.git`; worker Git traffic + uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/.git`. + Verify the broker remote and cleanly separate pre-existing changes, create a task branch, run the repository's tests, use `git push --dry-run` when proving access, and push a real branch only when the requested implementation is review-ready. Never @@ -318,11 +318,13 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For bounded + brokered Git for clone, fetch, and creation of a new namespaced feature + branch. Existing-ref updates, protected refs, deletion, and force-push are + rejected. For bounded repository/pull-request evidence or to create a review-ready draft PR, load - `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The - client injects the runtime Vault token without exposing it to the command - line, environment, output, or transcript. It permits only Atlas-scoped + `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The + client carries no repository credential; a separate least-authority + workload performs the allowed operation. It permits only Atlas-scoped metadata reads and verified same-repository draft creation. Updates, merge, approve, close, delete, comments, repository administration, and force-push are unavailable; never bypass the client with raw HTTP. Leave every PR @@ -330,14 +332,10 @@ data: verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in - `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the - existing cluster access to inspect the controller's job/build files and - logs rather than guessing a public hostname. The preferred read-only path - is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH] - [--commit SHA] --wait`; it distinguishes a genuinely terminal build from - nested Jenkins execution metadata and returns bounded JSON/log evidence. - With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. If Brad separately reports that he merged the PR, observe + `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use + Kubernetes pod logs and status evidence without exec or secret access. + Do not bypass the boundary to inspect controller files. If Brad separately + reports that he merged the PR, observe the default-branch build to a terminal result and report exact commit, build, and test evidence. @@ -349,19 +347,21 @@ data: The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the - relevant `titan-iac` manifests. Put durable desired-state changes on a + relevant `titan-iac` manifests. The broker namespace is deliberately + excluded from agent RBAC. Kubernetes cannot express a deny on one namespace + inside an all-namespace list, so enumerate namespaces and run namespaced + diagnostics instead of relying on `kubectl ... --all-namespaces`. Put + durable desired-state changes on a reviewable `titan-iac` branch and validate Kustomize and client dry-run. - Brad owns merge and routine Flux reconciliation; perform either only under - a separate explicit operator request. Direct `kubectl` mutations are for - explicit incident recovery or ephemeral verification, not ordinary - delivery. + Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not + available to the agent; hand an explicit incident mutation back to Brad. - Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any - current Kubernetes node name). Host keys are pinned and password fallback - is disabled. Use SSH only for host-level evidence or repairs that cannot be - performed through Kubernetes. Read first, identify the exact node and - impact, avoid fleet-wide destructive commands, and reflect persistent host - configuration in the appropriate tracked provisioning source. + Node SSH uses the dedicated locked-password `hermes-agent` OS account: + `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned, + password fallback is disabled, and the account has no sudo, disk, runtime, + or Kubernetes-storage authority. Use it for unprivileged host evidence; + hand privileged host repair back to Brad and reflect persistent changes in + the tracked provisioning source. Treat every credential, SSH identity, host-trust record, and credential-bearing access-client state as runtime-only Vault data. Keep @@ -398,7 +398,7 @@ data: [model]` for a persistent override. The first native Codex worker requires one device-code login; refreshed provider credentials persist through Vault while client caches remain disposable. The owner - workspace includes cluster-admin Kubernetes - access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, + workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`, + `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the browser/computer tools, and the native provider CLIs. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 4b93f92d..ccbc36ea 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -46,16 +46,6 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} - {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} @@ -186,9 +176,8 @@ spec: chmod 0600 "${profile_env}" chown 10000:10000 "${profile_env}" done - upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh upsert_env GIT_TERMINAL_PROMPT 0 - upsert_env GITEA_BASE_URL https://scm.bstein.dev + upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081 upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080 upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 @@ -664,6 +653,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -828,6 +818,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -897,6 +888,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1197,6 +1189,10 @@ spec: items: - {key: SKILL.md, path: SKILL.md} - {key: openai.yaml, path: agents/openai.yaml} + - name: scm-boundary + configMap: + name: hermes-scm-boundary + defaultMode: 0555 - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/agent-kubeconfig.yaml b/services/hermes/agent-kubeconfig.yaml index 15b38da9..9ad9242f 100644 --- a/services/hermes/agent-kubeconfig.yaml +++ b/services/hermes/agent-kubeconfig.yaml @@ -11,9 +11,9 @@ users: user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - - name: atlas-owner + - name: atlas-observer context: cluster: atlas user: hermes-agent namespace: default -current-context: atlas-owner +current-context: atlas-observer diff --git a/services/hermes/agent-rbac.yaml b/services/hermes/agent-rbac.yaml deleted file mode 100644 index 9dbff9e3..00000000 --- a/services/hermes/agent-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# services/hermes/agent-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-agent-cluster-admin - labels: - app.kubernetes.io/name: hermes-agent - app.kubernetes.io/part-of: hermes -subjects: - - kind: ServiceAccount - name: hermes-agent - namespace: hermes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 87984530..fe60bef9 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,13 +7,13 @@ images: digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 resources: - namespace.yaml + - scm-common - vault-serviceaccount.yaml - configmap.yaml - agent-configmap.yaml - chat-configmap.yaml - switchyard-configmap.yaml - rbac.yaml - - agent-rbac.yaml - node-ssh-access.yaml - pvc.yaml - switchyard-pvc.yaml @@ -69,8 +69,6 @@ configMapGenerator: - classifier_broker.py=scripts/classifier_broker.py - claude_oauth_broker.py=scripts/claude_oauth_broker.py - worker_route_broker.py=scripts/worker_route_broker.py - - gitea_api.py=scripts/gitea_api.py - - gitea_askpass.sh=scripts/gitea_askpass.sh - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py - hermes_stt_client.py=scripts/hermes_stt_client.py @@ -104,6 +102,14 @@ configMapGenerator: - config=agent-kubeconfig.yaml options: disableNameSuffixHash: true + - name: hermes-node-account-hardener + namespace: hermes + files: + - node_account_hardening.py=scripts/node_account_hardening.py + - node_account_audit.py=scripts/node_account_audit.py + - node_account_io.py=scripts/node_account_io.py + options: + disableNameSuffixHash: true - name: hermes-auto-router-plugin namespace: hermes files: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index d972889e..f9c3b09c 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -119,11 +119,65 @@ spec: app: server ports: - {protocol: TCP, port: 9010} - # agent.hermes.bstein.dev is an owner-only engineering workstation. The - # browser boundary remains OAuth-protected, while its workers need to reach - # every cluster namespace, Atlas LAN service, and hosted provider endpoint. egress: - - {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [gitea, hermes-scm] + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes-scm + podSelector: + matchLabels: + app: hermes-scm-broker + ports: + - {protocol: TCP, port: 9081} + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 192.168.0.0/16 + except: + - 192.168.22.9/32 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-node-ssh-access-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-node-ssh-access + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/node-ssh-access.yaml b/services/hermes/node-ssh-access.yaml index d9c6b223..04b0c78b 100644 --- a/services/hermes/node-ssh-access.yaml +++ b/services/hermes/node-ssh-access.yaml @@ -45,32 +45,16 @@ spec: - operator: Exists containers: - name: key-reconciler - image: busybox:1.37 + # The pinned index supports the cluster's amd64 and arm64 nodes. + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - reconcile() { - key="$(cat /vault/secrets/node-ssh-public-key)" - found=0 - for user in atlas oceanus; do - home="/host-home/${user}" - [ -d "${home}" ] || continue - found=1 - identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)" - [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")" - uid="${identity%%:*}" - gid="${identity##*:}" - install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh" - touch "${home}/.ssh/authorized_keys" - grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys" - chown "${uid}:${gid}" "${home}/.ssh/authorized_keys" - chmod 0600 "${home}/.ssh/authorized_keys" - done - [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; } - } while true; do - reconcile + /usr/local/bin/python \ + /opt/node-hardener/node_account_hardening.py \ + --public-key-file /vault/secrets/node-ssh-public-key sleep 300 done securityContext: @@ -83,8 +67,21 @@ spec: volumeMounts: - name: host-home mountPath: /host-home - - name: host-passwd - mountPath: /host-etc/passwd + - name: host-etc + mountPath: /host-etc + - name: host-polkit-share + mountPath: /host-polkit-share + readOnly: true + - name: host-k3s + mountPath: /host-k3s + - name: host-kubelet + mountPath: /host-kubelet + - name: host-run-k3s + mountPath: /host-run-k3s + - name: host-run-containerd + mountPath: /host-run-containerd + - name: coordinator + mountPath: /opt/node-hardener readOnly: true - name: vault-secrets mountPath: /vault/secrets @@ -94,19 +91,43 @@ spec: resources: requests: cpu: 5m - memory: 8Mi + memory: 32Mi limits: cpu: 50m - memory: 32Mi + memory: 128Mi volumes: - name: host-home hostPath: path: /home type: Directory - - name: host-passwd + - name: host-etc hostPath: - path: /etc/passwd - type: File + path: /etc + type: Directory + - name: host-polkit-share + hostPath: + path: /usr/share/polkit-1 + type: Directory + - name: host-k3s + hostPath: + path: /var/lib/rancher/k3s + type: Directory + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-run-k3s + hostPath: + path: /run/k3s + type: Directory + - name: host-run-containerd + hostPath: + path: /run/containerd + type: Directory + - name: coordinator + configMap: + name: hermes-node-account-hardener + defaultMode: 0555 - name: vault-secrets csi: driver: secrets-store.csi.k8s.io diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml new file mode 100644 index 00000000..17dc1da6 --- /dev/null +++ b/services/hermes/scm-common/kustomization.yaml @@ -0,0 +1,14 @@ +# services/hermes/scm-common/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: hermes-scm-boundary + files: + - gitea_api.py=scripts/gitea_api.py + - gitea_api_policy.py=scripts/gitea_api_policy.py + - scm_broker.py=scripts/scm_broker.py + - scm_broker_io.py=scripts/scm_broker_io.py + - scm_broker_server.py=scripts/scm_broker_server.py + - scm_broker_client.py=scripts/scm_broker_client.py + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py old mode 100755 new mode 100644 similarity index 65% rename from services/hermes/scripts/gitea_api.py rename to services/hermes/scm-common/scripts/gitea_api.py index b4a98eca..d207cc63 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -4,10 +4,10 @@ from __future__ import annotations import argparse +import base64 import json import os import re -import subprocess import sys import urllib.error import urllib.parse @@ -15,28 +15,29 @@ import urllib.request from collections.abc import Callable from pathlib import Path +from gitea_api_policy import ( + PolicyError, + _draft_title, + _reject_forbidden, + _validate_body, + _validate_pr_number, + _validate_pr_number_segment, + _validate_query, + _validate_ref, + _validate_ref_bounds, + _validate_repo, + _validate_sha, +) + CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" -DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") MAX_ERROR_BYTES = 8192 MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -DRAFT_TITLE_PREFIX = "WIP: " -GIT_BIN = "/usr/bin/git" -SENSITIVE_BODY_PATTERNS = ( - re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), - re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), - re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), - re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), - re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), -) - - -class PolicyError(ValueError): - """Raised when a requested Forgejo operation exceeds the safe boundary.""" +MAX_API_TARGET_LENGTH = 768 +MAX_API_PATH_LENGTH = 512 +RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -64,114 +65,48 @@ def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: def configured_base_url() -> str: """Reject attempts to redirect credentials away from the canonical origin.""" - configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL) if configured != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") return configured -def _validate_repo(repo: str) -> str: - if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: - raise PolicyError("repository name is outside the Atlas allowlist") - return repo - - -def _validate_ref(value: object, name: str) -> str: - """Validate the complete Git ref grammar using Git itself.""" - if not isinstance(value, str) or not value or value.startswith("-"): - raise PolicyError(f"{name} must be a same-repository branch name") - if any(ord(character) < 32 or ord(character) == 127 for character in value): - raise PolicyError(f"{name} must be a safe same-repository branch name") - result = subprocess.run( - [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - if result.returncode != 0: - raise PolicyError(f"{name} must be a safe same-repository branch name") - return value - - -def _validate_sha(value: object, name: str = "head SHA") -> str: - if not isinstance(value, str) or not SHA_RE.fullmatch(value): - raise PolicyError(f"{name} must be a full Git SHA-1") - return value.lower() - - -def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: - if not isinstance(value, str): - raise PolicyError(f"{name} must be text") - if required and not value.strip(): - raise PolicyError(f"{name} must not be empty") - if len(value) > maximum or "\x00" in value: - raise PolicyError(f"{name} exceeds the safe request limit") - return value - - -def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: - body = _validate_text(value, "body", 16384, required=False) - if any(secret and secret in body for secret in forbidden): - raise PolicyError("pull-request body contains runtime credential material") - if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): - raise PolicyError("pull-request body resembles credential material") - return body - - -def _draft_title(value: object) -> str: - """Return a bounded title using Gitea's configured default draft prefix.""" - title = _validate_text(value, "title", 251, required=True).strip() - for prefix in ("WIP:", "[WIP]"): - if title.upper().startswith(prefix): - title = title[len(prefix) :].lstrip() - break - if not title: - raise PolicyError("title must contain text after the draft prefix") - return DRAFT_TITLE_PREFIX + title - - def _split_api_path(path: str) -> urllib.parse.SplitResult: + if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH: + raise PolicyError("API target exceeds the safe size limit") + if ( + not path.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or "\\" in path + or not RAW_API_TARGET_RE.fullmatch(path) + ): + raise PolicyError("API target contains non-canonical raw characters") target = urllib.parse.urlsplit(path) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != path: + raise PolicyError("API target is not in exact canonical form") if target.scheme or target.netloc or target.fragment: raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): raise PolicyError("API path must start with /api/v1/") - if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + if len(target.path) > MAX_API_PATH_LENGTH: + raise PolicyError("API path exceeds the safe size limit") + segments = target.path.split("/") + if "//" in target.path or any(segment in {".", ".."} for segment in segments): raise PolicyError("encoded or non-canonical API paths are not allowed") return target -def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: - try: - pairs = urllib.parse.parse_qsl( - target.query, keep_blank_values=True, strict_parsing=True - ) - except ValueError as exc: - raise PolicyError("invalid API query") from exc - if len({key for key, _ in pairs}) != len(pairs): - raise PolicyError("duplicate API query parameters are not allowed") - if any(key not in allowed for key, _ in pairs): - raise PolicyError("API query parameter is outside the read allowlist") - values = dict(pairs) - for name in ("page", "limit"): - if name not in values: - continue - if not values[name].isdigit() or int(values[name]) < 1: - raise PolicyError(f"{name} must be a positive integer") - if "limit" in values and int(values["limit"]) > 50: - raise PolicyError("read limit cannot exceed 50") - if "state" in values and values["state"] not in {"open", "closed", "all"}: - raise PolicyError("pull-request state is invalid") - - -def _authorize_read(target: urllib.parse.SplitResult) -> str: +def _authorize_read( + target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = () +) -> str: """Allow only repository, PR, branch, commit, and status metadata reads.""" prefix = "/api/v1/repos/atlas/" remainder = target.path.removeprefix(prefix) if remainder == target.path: raise PolicyError("reads are limited to explicit Atlas repository metadata") repo, separator, suffix = remainder.partition("/") + _reject_forbidden(repo, "repository", forbidden) _validate_repo(repo) if not separator: @@ -180,10 +115,18 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: if suffix == "pulls": _validate_query(target, {"page", "limit", "state"}) return "pull-list" - if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix) + if content_match: + _validate_pr_number_segment(content_match.group(1)) + raise PolicyError("repository API route is outside the metadata read allowlist") + pull_match = re.fullmatch(r"pulls/([^/]+)", suffix) + if pull_match: + _validate_pr_number_segment(pull_match.group(1)) _validate_query(target, set()) return "pull" - if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix) + if evidence_match: + _validate_pr_number_segment(evidence_match.group(1)) _validate_query(target, {"page", "limit"}) return "pull-evidence" if suffix == "branches": @@ -211,7 +154,7 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" - if base_url.rstrip("/") != CANONICAL_BASE_URL: + if base_url != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") target = _split_api_path(path) return urllib.parse.urlunsplit( @@ -219,14 +162,20 @@ def api_url(base_url: str, path: str) -> str: ) -def authorize_request(method: str, path: str, data: object | None) -> str: +def authorize_request( + method: str, + path: str, + data: object | None, + *, + forbidden: tuple[str, ...] = (), +) -> str: """Validate one request and return its bounded operation name.""" normalized_method = method.upper() target = _split_api_path(path) if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - return _authorize_read(target) + return _authorize_read(target, forbidden=forbidden) if target.query: raise PolicyError("mutating operations cannot include query parameters") if normalized_method != "POST": @@ -234,14 +183,17 @@ def authorize_request(method: str, path: str, data: object | None) -> str: match = CREATE_PATH_RE.fullmatch(target.path) if not match: raise PolicyError("POST is limited to creating an Atlas draft pull request") + _reject_forbidden(match.group(1), "repository", forbidden) _validate_repo(match.group(1)) if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): + if data["title"] != _draft_title(data["title"], forbidden=forbidden): raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_body(data["body"]) + _validate_body(data["body"], forbidden=forbidden) + _validate_ref_bounds(data["base"], "base", forbidden=forbidden) + _validate_ref_bounds(data["head"], "head", forbidden=forbidden) + _validate_ref(data["base"], "base", forbidden=forbidden) + _validate_ref(data["head"], "head", forbidden=forbidden) return "create-draft" @@ -254,7 +206,19 @@ def build_request( data: object | None = None, ) -> urllib.request.Request: """Build an authorized request without putting the token in its URL or body.""" - authorize_request(method, path, data) + authorize_request(method, path, data, forbidden=(token,)) + return _build_request(method, path, base_url=base_url, token=token, data=data) + + +def _build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build one already-authorized upstream request.""" payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") @@ -281,6 +245,19 @@ def redact_bytes(value: bytes, token: str) -> bytes: ) +def _reject_response_credential(value: bytes, token: str) -> None: + """Fail closed if an upstream body reflects any ordinary token encoding.""" + raw = token.encode("utf-8") + forms = { + raw, + base64.b64encode(raw), + base64.b64encode(b"hermes-automation:" + raw), + urllib.parse.quote(token, safe="").encode("ascii"), + } + if any(form and form in value for form in forms): + raise PolicyError("Forgejo response contains runtime credential material") + + def _request( method: str, path: str, @@ -288,14 +265,23 @@ def _request( *, token: str, opener: Callable[..., object] = _safe_urlopen, + authorized: bool = False, + expected_status: int, ) -> bytes: - request = build_request( - method, path, base_url=configured_base_url(), token=token, data=data - ) + builder = _build_request if authorized else build_request + request = builder(method, path, base_url=configured_base_url(), token=token, data=data) with opener(request, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != expected_status: + raise PolicyError("Forgejo returned an unexpected HTTP status") + if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined] + raise PolicyError("Forgejo returned an unexpected response type") body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] if len(body) > MAX_RESPONSE_BYTES: raise PolicyError("Forgejo response exceeds the safe size limit") + _reject_response_credential(body, token) return redact_bytes(body, token) @@ -303,7 +289,9 @@ def read( path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: """Read one explicitly allowed Atlas repository metadata resource.""" - return _request("GET", path, None, token=token, opener=opener) + return _request( + "GET", path, None, token=token, opener=opener, expected_status=200 + ) def _nested(document: dict[str, object], *keys: str) -> object: @@ -332,9 +320,7 @@ def _require_create_response( raise PolicyError("Forgejo returned invalid pull-request metadata") from exc if not isinstance(document, dict): raise PolicyError("Forgejo returned invalid pull-request metadata") - number = document.get("number") - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("Forgejo omitted a valid pull-request number") + number = _validate_pr_number(document.get("number")) full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" checks = ( @@ -368,12 +354,15 @@ def create_draft( opener: Callable[..., object] = _safe_urlopen, ) -> bytes: """Create and verify a same-repository draft pull request for human review.""" + _reject_forbidden(repo, "repository", (token,)) repo = _validate_repo(repo) - base = _validate_ref(base, "base") - head = _validate_ref(head, "head") - head_sha = _validate_sha(head_sha) - title = _draft_title(title) + title = _draft_title(title, forbidden=(token,)) body = _validate_body(body, forbidden=(token,)) + head_sha = _validate_sha(head_sha) + _validate_ref_bounds(base, "base", forbidden=(token,)) + _validate_ref_bounds(head, "head", forbidden=(token,)) + base = _validate_ref(base, "base", forbidden=(token,)) + head = _validate_ref(head, "head", forbidden=(token,)) data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", @@ -381,6 +370,8 @@ def create_draft( data, token=token, opener=opener, + authorized=True, + expected_status=201, ) _require_create_response( result, @@ -417,16 +408,24 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: +def _operation( + args: argparse.Namespace, *, validate_refs: bool = True +) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) _validate_sha(args.head_sha) + body = _validate_body(args.body) + title = _draft_title(args.title) + _validate_ref_bounds(args.base, "base") + _validate_ref_bounds(args.head, "head") + base = _validate_ref(args.base, "base") if validate_refs else args.base + head = _validate_ref(args.head, "head") if validate_refs else args.head data = { - "base": _validate_ref(args.base, "base"), - "body": _validate_body(args.body), - "head": _validate_ref(args.head, "head"), - "title": _draft_title(args.title), + "base": base, + "body": body, + "head": head, + "title": title, } return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data @@ -440,12 +439,11 @@ def _write_body(body: bytes) -> None: def main(argv: list[str] | None = None) -> int: """Validate, optionally describe, and execute one safe Forgejo operation.""" - token = "" try: args = parse_args(argv) - method, path, data = _operation(args) - operation = authorize_request(method, path, data) if args.dry_run: + method, path, data = _operation(args, validate_refs=False) + operation = authorize_request(method, path, data) print( json.dumps( { @@ -462,29 +460,27 @@ def main(argv: list[str] | None = None) -> int: ) ) return 0 - token = read_token() + method, path, data = _operation(args, validate_refs=False) + from scm_broker_client import create_draft as broker_create_draft + from scm_broker_client import read as broker_read + if args.operation == "read": - body = read(path, token=token) + body = broker_read(path) else: assert isinstance(data, dict) - body = create_draft( + body = broker_create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), - token=token, ) _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" - print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) - if body: - sys.stderr.buffer.write(body) - if not body.endswith(b"\n"): - sys.stderr.buffer.write(b"\n") + exc.read(MAX_ERROR_BYTES) + print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr) return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): print( diff --git a/services/hermes/scm-common/scripts/gitea_api_policy.py b/services/hermes/scm-common/scripts/gitea_api_policy.py new file mode 100644 index 00000000..db8833cf --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api_policy.py @@ -0,0 +1,490 @@ +"""Validation policy for Hermes' least-authority Atlas Forgejo client. + +The screening catches structured credential assignments, known token formats, +and long high-entropy values. It is a fail-closed accident barrier, not proof +that text is secret-free: a short or multiword secret under an innocuous key is +not reliably distinguishable from prose and must never be supplied by callers. +JSON depth/nodes/documents and multiline gaps are deliberately bounded; large +or unusual configuration blobs are not a supported pull-request field format. +""" + +# ruff: noqa: SIM905 + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +from collections import Counter +from math import log2 + +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +MAX_QUERY_LENGTH = 128 +MAX_QUERY_ITEMS = 3 +MAX_QUERY_KEY_LENGTH = 16 +MAX_QUERY_VALUE_LENGTH = 16 +MAX_PAGE = 10_000 +MAX_LIMIT = 50 +MAX_PR_NUMBER = 2_147_483_647 +MAX_PR_NUMBER_DIGITS = 10 +MAX_REF_CHARACTERS = 200 +MAX_REF_UTF8_BYTES = 255 +MAX_TITLE_UTF8_BYTES = 512 +MAX_BODY_UTF8_BYTES = 32_768 +MAX_JSON_DOCUMENTS = 32 +MAX_JSON_NODES = 2_048 +MAX_JSON_DEPTH = 32 +CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z") +ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?[\"']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]*(?P[:=])[ \t]* + (?P + \"(?:\\.|[^\"\\\r\n])*\" | + '(?:\\.|[^'\\\r\n])*' | + [^\r\n,;}&]{0,2048} + ) + """ +) +JSON_KEY_RE = re.compile(r'"(?P[^"]{1,512})"(?P[\x00-\x20\x7f]{0,256}):') +MULTILINE_ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?["']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4} + (?P[:=]) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4} + (?P[^\r\n,;}&]{1,2048}) + """ +) +CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +CREDENTIAL_SCHEME_RE = re.compile( + r"(?i)\b(?:bearer|basic|token)\s+" + r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])" + r"[A-Za-z0-9+/_=.:-]{8,}" +) +STANDALONE_SECRET_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"(? str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None: + """Reject exact runtime credentials before invoking helpers or the network.""" + if not isinstance(value, str) or any(secret and secret in value for secret in forbidden): + raise PolicyError(f"{name} contains runtime credential material") + + +def _validate_ref_bounds( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Reject oversized or non-encodable refs before invoking Git.""" + _reject_forbidden(value, name, forbidden) + if not value or len(value) > MAX_REF_CHARACTERS: + raise PolicyError(f"{name} exceeds the safe branch-name limit") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(encoded) > MAX_REF_UTF8_BYTES: + raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit") + return value + + +def _validate_ref( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Validate the complete Git ref grammar using Git itself.""" + value = _validate_ref_bounds(value, name, forbidden=forbidden) + if value.startswith("-"): + raise PolicyError(f"{name} must be a same-repository branch name") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + +def _validate_pr_number_segment(value: object) -> int: + """Validate a canonical bounded pull-request number from an API path.""" + if ( + not isinstance(value, str) + or len(value) > MAX_PR_NUMBER_DIGITS + or not re.fullmatch(r"[1-9][0-9]*", value) + ): + raise PolicyError("pull-request number must use bounded canonical ASCII digits") + number = int(value) + if number > MAX_PR_NUMBER: + raise PolicyError("pull-request number is outside its allowed range") + return number + + +def _validate_pr_number(value: object) -> int: + """Validate a bounded pull-request number returned by Forgejo.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= MAX_PR_NUMBER + ): + raise PolicyError("Forgejo omitted a valid pull-request number") + return value + + +def _validate_text( + value: object, + name: str, + maximum: int, + maximum_bytes: int, + *, + required: bool, +) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _normalized_key(value: str) -> tuple[tuple[str, ...], str]: + """Split camelCase and separator-based assignment keys into semantics.""" + separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'")) + words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower())) + return words, "".join(words) + + +def _is_sensitive_key(value: str) -> bool: + words, compact = _normalized_key(value) + word_set = set(words) + if word_set & SENSITIVE_KEY_WORDS: + return True + if compact in SENSITIVE_COMPACT_KEYS: + return True + if compact.endswith(SENSITIVE_COMPACT_SUFFIXES): + return True + if "key" in word_set and word_set & KEY_MODIFIER_WORDS: + return True + if {"client", "email"} <= word_set or {"client", "id"} <= word_set: + return True + if {"access", "id"} <= word_set or {"connection", "string"} <= word_set: + return True + return compact.endswith("configjson") + + +def _strip_assignment_value(value: str) -> tuple[str, bool]: + stripped = value.strip() + quoted = ( + len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0] + ) + if quoted: + stripped = stripped[1:-1].strip() + return stripped, quoted + + +def _looks_like_prose(value: str) -> bool: + words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value) + if len(words) < 2: + return False + if re.search(r"[$`{}\[\]\\@/:=+_]", value): + return False + return words[0].lower() in PROSE_LEAD_WORDS + + +def _looks_sensitive_assignment_value(value: str) -> bool: + stripped, quoted = _strip_assignment_value(value) + if not stripped: + return False + if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")): + return True + if CREDENTIAL_SCHEME_RE.search(stripped): + return True + if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS): + return True + if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped): + return True + if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped): + return True + return not _looks_like_prose(stripped) + + +def _is_structural_credential_assignment(key: str, value: str) -> bool: + words, compact_key = _normalized_key(key) + stripped, _quoted = _strip_assignment_value(value) + _value_words, compact_value = _normalized_key(stripped) + return ( + compact_key == "type" and compact_value in {"serviceaccount", "credential"} + ) or ("service" in words and "account" in words and bool(stripped)) + + +def _decode_json_key(raw: str) -> tuple[str, bool]: + """Decode one bounded JSON key without silently accepting bad escapes.""" + try: + decoded = json.loads(f'"{raw}"') + except (json.JSONDecodeError, UnicodeError): + approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", "")) + return approximate, False + return decoded if isinstance(decoded, str) else "", True + + +def _json_value_has_sensitive_assignment( + value: object, *, depth: int = 0, nodes: list[int] | None = None +) -> bool: + """Walk one decoded JSON value with explicit depth and node ceilings.""" + if nodes is None: + nodes = [0] + nodes[0] += 1 + if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + raise PolicyError("structured pull-request text exceeds the scan limit") + if isinstance(value, dict): + for key, assigned in value.items(): + if not isinstance(key, str): + continue + if _is_sensitive_key(key): + return True + assigned_text = assigned if isinstance(assigned, str) else "" + if _is_structural_credential_assignment(key, assigned_text): + return True + if _json_value_has_sensitive_assignment( + assigned, depth=depth + 1, nodes=nodes + ): + return True + elif isinstance(value, list): + return any( + _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes) + for item in value + ) + return False + + +def _decoded_json_documents(value: str): + """Yield bounded embedded JSON object or array fragments.""" + decoder = json.JSONDecoder() + index = 0 + attempts = 0 + while attempts < MAX_JSON_DOCUMENTS: + positions = [ + position for token in "{[" if (position := value.find(token, index)) >= 0 + ] + if not positions: + return + start = min(positions) + attempts += 1 + try: + document, end = decoder.raw_decode(value, start) + except (json.JSONDecodeError, RecursionError, ValueError): + index = start + 1 + continue + yield document + index = max(end, start + 1) + if "{" in value[index:] or "[" in value[index:]: + raise PolicyError("structured pull-request text exceeds the scan limit") + + +def _has_structured_sensitive_assignment(value: str) -> bool: + """Detect bounded JSON and multiline YAML/env credential assignments.""" + for candidate in JSON_KEY_RE.finditer(value): + decoded, valid = _decode_json_key(candidate.group("key")) + if _is_sensitive_key(decoded): + return True + if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")): + return True + for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + return True + return any( + _json_value_has_sensitive_assignment(item) + for item in _decoded_json_documents(value) + ) + + +def _has_high_entropy_token(value: str) -> bool: + for match in HIGH_ENTROPY_TOKEN_RE.finditer(value): + token = match.group(0) + if len(token) > 256: + return True + groups = sum( + bool(re.search(pattern, token)) + for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]") + ) + counts = Counter(token) + entropy = -sum( + (count / len(token)) * log2(count / len(token)) for count in counts.values() + ) + if groups >= 3 and entropy >= 4.0: + return True + return False + + +def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None: + if any(secret and secret in value for secret in forbidden): + raise PolicyError(f"pull-request {name} contains runtime credential material") + if CREDENTIAL_SCHEME_RE.search(value): + raise PolicyError(f"pull-request {name} resembles credential material") + if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_structured_sensitive_assignment(value): + raise PolicyError(f"pull-request {name} resembles credential material") + for candidate in ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_high_entropy_token(value): + raise PolicyError(f"pull-request {name} resembles credential material") + + +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False) + _reject_sensitive(body, "body", forbidden) + return body + + +def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + """Return a bounded, secret-screened Gitea draft title.""" + title = _validate_text( + value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True + ).strip() + _reject_sensitive(title, "title", forbidden) + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + """Accept only short canonical ASCII query strings with bounded pagination.""" + raw = target.query + if ( + len(raw) > MAX_QUERY_LENGTH + or not raw.isascii() + or not CANONICAL_QUERY_RE.fullmatch(raw) + or "%" in raw + ): + raise PolicyError("API query must use short canonical ASCII form") + try: + pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len(pairs) > MAX_QUERY_ITEMS: + raise PolicyError("too many API query parameters") + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any( + not key + or len(key) > MAX_QUERY_KEY_LENGTH + or len(value) > MAX_QUERY_VALUE_LENGTH + for key, value in pairs + ): + raise PolicyError("API query key or value exceeds its safe limit") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not re.fullmatch(r"[0-9]+", values[name]): + raise PolicyError(f"{name} must use ASCII decimal digits") + number = int(values[name]) + if values[name] != str(number): + raise PolicyError(f"{name} must use canonical ASCII decimal form") + ceiling = MAX_PAGE if name == "page" else MAX_LIMIT + if not 1 <= number <= ceiling: + raise PolicyError(f"{name} is outside its allowed range") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py new file mode 100644 index 00000000..a0c4043c --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import socket +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler +from typing import BinaryIO + +from gitea_api import ( + CANONICAL_BASE_URL, + PolicyError, + create_draft, + read, + read_token, +) +from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo +from scm_broker_io import RejectRedirect, response_status as _status, spool_response +from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer + +BROKER_PORT = 9081 +GIT_USER = "hermes-automation" +MAX_CONTROL_BODY = 64 * 1024 +MAX_GIT_REQUEST = 128 * 1024 * 1024 +MAX_GIT_RESPONSE = 128 * 1024 * 1024 +MAX_PUSH_COMMANDS = 16 +MAX_HEADERS = 32 +MAX_HEADER_BYTES = 16 * 1024 +SPOOL_MEMORY_LIMIT = 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +INBOUND_HEADER_TIMEOUT = 10.0 +INBOUND_BODY_TIMEOUT = 120.0 +ZERO_SHA = b"0" * 40 +GIT_PATH_RE = re.compile( + r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" + r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" +) +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) + + +UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect()) + + +def _read_bounded( + stream, + maximum: int, + length: int | None = None, + *, + deadline: float | None = None, + set_timeout=None, +) -> bytes: + if length is not None and not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + wanted = maximum + 1 if length is None else length + chunks: list[bytes] = [] + total = 0 + while total < wanted: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining) + chunk = stream.read(min(STREAM_CHUNK, wanted - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + value = b"".join(chunks) + if len(value) > maximum or (length is not None and len(value) != length): + raise PolicyError("SCM request exceeds the safe size limit") + return value + + +def _spool_bounded( + stream, + maximum: int, + length: int, + *, + token: str, + context: str, + deadline: float, + set_timeout=None, +) -> tuple[BinaryIO, int]: + """Copy a fixed-length exchange through a bounded-memory disk spool.""" + if not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + remaining = length + forms = _credential_forms(token) + carry = b"" + try: + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining_time) + chunk = stream.read(min(STREAM_CHUNK, remaining)) + if not chunk: + raise PolicyError("SCM request body ended early") + candidate = carry + chunk + if any(form in candidate for form in forms): + raise PolicyError(f"{context} contains runtime credential material") + width = max(len(form) for form in forms) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + remaining -= len(chunk) + spool.seek(0) + return spool, length + except Exception: + spool.close() + raise + + +def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]: + return spool_response( + response, + maximum, + _credential_forms(token), + memory_limit=SPOOL_MEMORY_LIMIT, + chunk_size=STREAM_CHUNK, + deadline_seconds=INBOUND_BODY_TIMEOUT, + ) + + +def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]: + if handler.headers.get("Transfer-Encoding"): + raise PolicyError("chunked broker control requests are not allowed") + if handler.headers.get_content_type() != "application/json": + raise PolicyError("broker control request must be JSON") + length = _content_length(handler.headers, MAX_CONTROL_BODY) + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + value = json.loads( + _read_bounded( + handler.rfile, + MAX_CONTROL_BODY, + length, + deadline=deadline, + set_timeout=handler.connection.settimeout, + ) + ) + if not isinstance(value, dict): + raise PolicyError("broker control request must be an object") + return value + + +def _git_target(raw: str) -> tuple[str, str, str]: + if ( + not raw.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in raw) + or "%" in raw + or "\\" in raw + or len(raw) > 512 + ): + raise PolicyError("Git target is not canonical ASCII") + target = urllib.parse.urlsplit(raw) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != raw: + raise PolicyError("Git target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("Git target must be relative") + match = GIT_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("Git target is outside the Atlas allowlist") + repo = _validate_repo(match.group("repo")) + operation = match.group("operation") + query = target.query + if operation == "info/refs": + if query not in {"service=git-upload-pack", "service=git-receive-pack"}: + raise PolicyError("Git discovery service is outside the allowlist") + service = query.removeprefix("service=") + elif query: + raise PolicyError("Git RPC query parameters are not allowed") + else: + service = operation + return repo, operation, service + + +def _content_length(headers: object, maximum: int) -> int: + """Parse one short canonical bounded Content-Length header.""" + raw = headers.get("Content-Length", "") # type: ignore[attr-defined] + if ( + not isinstance(raw, str) + or not raw.isascii() + or len(raw) > 10 + or not raw.isdigit() + ): + raise PolicyError("SCM request length is invalid") + length = int(raw) + if raw != str(length) or not 0 <= length <= maximum: + raise PolicyError("SCM request length is invalid") + return length + + +def _credential_forms(token: str) -> tuple[bytes, ...]: + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) + return token.encode("utf-8"), basic, b"Basic " + basic + + +def _reject_credential_bytes(value: bytes, token: str, context: str) -> None: + if any(form in value for form in _credential_forms(token)): + raise PolicyError(f"{context} contains runtime credential material") + + +def _receive_prefix(body: bytes | BinaryIO) -> bytes: + if isinstance(body, bytes): + return body + position = body.tell() + try: + body.seek(0) + return body.read(64 * 1024) + finally: + body.seek(position) + + +def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: + """Permit only creation of new, namespaced feature branches.""" + body = _receive_prefix(body) + _reject_credential_bytes(body, token, "Git request") + position = 0 + commands = 0 + while position + 4 <= len(body): + header = body[position : position + 4] + if not re.fullmatch(rb"[0-9a-f]{4}", header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + return + if size < 4 or position + size > len(body): + raise PolicyError("Git receive-pack command framing is invalid") + command = body[position + 4 : position + size].rstrip(b"\n") + command = command.split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + position += size + raise PolicyError("Git receive-pack request omitted its command terminator") + + +def _upstream_git_request( + target: str, + *, + method: str, + body: bytes | BinaryIO | None, + content_type: str | None, + expected_type: str, + token: str, + body_length: int | None = None, + stream_result: bool = False, + opener=UPSTREAM_OPENER.open, +) -> bytes | tuple[BinaryIO, int]: + credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") + headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} + if content_type: + headers["Content-Type"] = content_type + if body is not None: + if body_length is None: + if not isinstance(body, bytes): + raise PolicyError("Git upstream request length is missing") + body_length = len(body) + headers["Content-Length"] = str(body_length) + request = urllib.request.Request( + CANONICAL_BASE_URL + target, + data=body, + method=method, + headers=headers, + ) + with opener(request, timeout=120) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token) + if stream_result: + return result, result_length + try: + return result.read() + finally: + result.close() + + +class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): + """Expose only bounded metadata, draft creation, and smart-HTTP Git.""" + + server_version = "HermesSCMBroker/1" + sys_version = "" + header_deadline_seconds = INBOUND_HEADER_TIMEOUT + + def _validate_headers(self) -> None: + items = list(self.headers.items()) + if len(items) > MAX_HEADERS: + raise PolicyError("SCM request has too many headers") + total = 0 + for name, value in items: + if not name.isascii() or not value.isascii(): + raise PolicyError("SCM request headers must be ASCII") + total += len(name) + len(value) + 4 + if any(ord(character) < 32 and character != "\t" for character in value): + raise PolicyError("SCM request header contains controls") + if total > MAX_HEADER_BYTES: + raise PolicyError("SCM request headers exceed the safe limit") + if len(self.headers.get_all("Content-Length", [])) > 1: + raise PolicyError("SCM request has duplicate Content-Length") + + def _stream( + self, status: int, content_type: str, body: BinaryIO, length: int + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.end_headers() + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + while True: + chunk = body.read(STREAM_CHUNK) + if not chunk: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM response write deadline exceeded") + self.connection.settimeout(remaining) + self.wfile.write(chunk) + + def log_message(self, _format: str, *_args: object) -> None: + return + + def _json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _reject(self, status: int = 400) -> None: + self._json(status, b'{"error":"request rejected"}') + + def do_GET(self) -> None: + try: + self._validate_headers() + if self.path == "/healthz": + self._json(200, b'{"status":"ok"}') + return + repo, operation, service = _git_target(self.path) + if operation != "info/refs": + raise PolicyError("Git RPC requires POST") + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + expected = f"application/x-{service}-advertisement" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/info/refs?service={service}", + method="GET", + body=None, + content_type=None, + expected_type=expected, + token=token, + stream_result=True, + ) + body, length = streamed + try: + self._stream(200, expected, body, length) + finally: + body.close() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + ): + self._reject() + + def do_POST(self) -> None: + try: + self._validate_headers() + self.connection.settimeout(INBOUND_BODY_TIMEOUT) + if self.path in {"/v1/metadata", "/v1/drafts"}: + self._control() + else: + self._git_rpc() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): + self._reject() + + def _control(self) -> None: + data = _load_json(self) + token = read_token() + if self.path == "/v1/metadata": + if set(data) != {"path"} or not isinstance(data["path"], str): + raise PolicyError("metadata request fields are invalid") + result = read(data["path"], token=token) + else: + expected = {"base", "body", "head", "head_sha", "repo", "title"} + if set(data) != expected or not all( + isinstance(data[key], str) for key in expected + ): + raise PolicyError("draft request fields are invalid") + result = create_draft(token=token, **data) # type: ignore[arg-type] + if token.encode("utf-8") in result: + raise PolicyError("SCM upstream reflected credential material") + self._json(200, result) + + def _git_rpc(self) -> None: + repo, operation, service = _git_target(self.path) + if ( + operation not in {"git-upload-pack", "git-receive-pack"} + or service != operation + ): + raise PolicyError("Git RPC operation is outside the allowlist") + expected_request = f"application/x-{service}-request" + if self.headers.get_content_type() != expected_request or self.headers.get( + "Transfer-Encoding" + ): + raise PolicyError("Git RPC request type is invalid") + length = _content_length(self.headers, MAX_GIT_REQUEST) + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + body, body_length = _spool_bounded( + self.rfile, + MAX_GIT_REQUEST, + length, + token=token, + context="Git request", + deadline=time.monotonic() + INBOUND_BODY_TIMEOUT, + set_timeout=self.connection.settimeout, + ) + try: + if service == "git-receive-pack": + _validate_receive_pack(body, token) + expected = f"application/x-{service}-result" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/{service}", + method="POST", + body=body, + body_length=body_length, + content_type=expected_request, + expected_type=expected, + token=token, + stream_result=True, + ) + result, result_length = streamed + try: + self._stream(200, expected, result, result_length) + finally: + result.close() + finally: + body.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="0.0.0.0") + parser.add_argument("--port", type=int, default=BROKER_PORT) + args = parser.parse_args() + BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py new file mode 100644 index 00000000..6c57ddfd --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin.""" + +from __future__ import annotations + +import json +import urllib.request +from collections.abc import Callable + +from gitea_api_policy import PolicyError + +BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Keep every broker request on its fixed in-cluster origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM broker redirects are not allowed") + + +_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _open(request: urllib.request.Request, timeout: int): + return _OPENER.open(request, timeout=timeout) + + +def request( + endpoint: str, + payload: dict[str, object], + *, + opener: Callable[..., object] = _open, +) -> bytes: + """Send one bounded broker operation without any repository credential.""" + if endpoint not in {"/v1/metadata", "/v1/drafts"}: + raise PolicyError("SCM broker operation is outside the client allowlist") + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + if len(body) > 64 * 1024: + raise PolicyError("SCM broker request exceeds the safe size limit") + outgoing = urllib.request.Request( + BROKER_ORIGIN + endpoint, + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "hermes-scm-broker-client/1", + }, + ) + with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != 200: + raise PolicyError("SCM broker returned an unexpected HTTP status") + content_type = response.headers.get_content_type() # type: ignore[attr-defined] + if content_type != "application/json": + raise PolicyError("SCM broker returned an unexpected response type") + result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(result) > MAX_RESPONSE_BYTES: + raise PolicyError("SCM broker response exceeds the safe size limit") + json.loads(result) + return result + + +def read(path: str, *, opener: Callable[..., object] = _open) -> bytes: + """Read explicitly allowed Atlas metadata through the broker.""" + return request("/v1/metadata", {"path": path}, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + opener: Callable[..., object] = _open, +) -> bytes: + """Create one verified draft through the broker for human review.""" + return request( + "/v1/drafts", + { + "base": base, + "body": body, + "head": head, + "head_sha": head_sha, + "repo": repo, + "title": title, + }, + opener=opener, + ) diff --git a/services/hermes/scm-common/scripts/scm_broker_io.py b/services/hermes/scm-common/scripts/scm_broker_io.py new file mode 100644 index 00000000..96df1fe0 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_io.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Bounded-memory stream helpers for the Hermes SCM broker.""" + +from __future__ import annotations + +import tempfile +import time +import urllib.request +from typing import BinaryIO + +from gitea_api_policy import PolicyError + + +class RejectRedirect(urllib.request.HTTPRedirectHandler): + """Reject every upstream redirect before credentials can be forwarded.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM upstream redirects are not allowed") + + +def response_status(response: object) -> int | None: + """Read an HTTP status from either urllib response interface.""" + value = getattr(response, "status", None) + if value is None and hasattr(response, "getcode"): + value = response.getcode() # type: ignore[attr-defined] + return value + + +def spool_response( + response, + maximum: int, + forbidden: tuple[bytes, ...], + *, + memory_limit: int, + chunk_size: int, + deadline_seconds: float, +) -> tuple[BinaryIO, int]: + """Spool an upstream response to disk while scanning chunk boundaries.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=memory_limit, dir="/tmp" + ) + carry = b"" + total = 0 + deadline = time.monotonic() + deadline_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM upstream response deadline exceeded") + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise PolicyError("SCM upstream response exceeds the safe size limit") + candidate = carry + chunk + if any(value in candidate for value in forbidden): + raise PolicyError( + "Git upstream response contains runtime credential material" + ) + width = max(len(value) for value in forbidden) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + spool.seek(0) + return spool, total + except Exception: + spool.close() + raise diff --git a/services/hermes/scm-common/scripts/scm_broker_server.py b/services/hermes/scm-common/scripts/scm_broker_server.py new file mode 100644 index 00000000..f29cd314 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker.""" + +from __future__ import annotations + +import threading +import time +from http.server import ThreadingHTTPServer + +MAX_CONCURRENT_REQUESTS = 3 + + +class _AbsoluteDeadlineReader: + """Read header lines against one wall-clock deadline, not idle timeouts.""" + + def __init__(self, stream, connection): + self._stream = stream + self._connection = connection + self._deadline: float | None = None + + def begin(self, timeout: float) -> None: + self._deadline = time.monotonic() + timeout + + def end(self) -> None: + self._deadline = None + + def readline(self, limit: int = -1) -> bytes: + if self._deadline is None: + return self._stream.readline(limit) + value = bytearray() + while limit < 0 or len(value) < limit: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute SCM header deadline exceeded") + self._connection.settimeout(remaining) + character = self._stream.read(1) + if not character: + break + value.extend(character) + if character == b"\n": + break + return bytes(value) + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class AbsoluteHeaderDeadlineMixin: + """Apply an absolute request-line plus header deadline to HTTP handlers.""" + + header_deadline_seconds = 10.0 + + def setup(self) -> None: + super().setup() + self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection) + self.rfile = self._header_reader + + def handle_one_request(self) -> None: + try: + self._header_reader.begin(self.header_deadline_seconds) + try: + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = "" + self.request_version = "" + self.command = "" + self.send_error(414) + return + if not self.raw_requestline: + self.close_connection = True + return + if not self.parse_request(): + return + finally: + self._header_reader.end() + method_name = "do_" + self.command + if not hasattr(self, method_name): + self.send_error(501, "Unsupported method") + return + getattr(self, method_name)() + self.wfile.flush() + except TimeoutError: + self.close_connection = True + + +class BoundedThreadingHTTPServer(ThreadingHTTPServer): + """Cap active handlers so large Git exchanges cannot exhaust memory.""" + + daemon_threads = True + block_on_close = True + request_queue_size = 16 + + def __init__(self, *args, **kwargs): + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS) + super().__init__(*args, **kwargs) + + def process_request(self, request, client_address) -> None: + if not self._slots.acquire(blocking=False): + try: + request.sendall( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"Connection: close\r\nContent-Length: 0\r\n\r\n" + ) + finally: + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + self.shutdown_request(request) + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh deleted file mode 100755 index d6146d80..00000000 --- a/services/hermes/scripts/gitea_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s /runtime-access/gitea-username ]; then - tr -d '\r\n' Path: @@ -139,24 +141,8 @@ def bootstrap_cassandra_state(root: Path) -> dict[str, str]: return {"state": "ready"} -def _gitea_token_path() -> Path: - """Return the runtime-only Gitea credential path without reading it.""" - configured = os.environ.get("HERMES_GITEA_TOKEN_FILE", "").strip() - return Path(configured) if configured else DEFAULT_GITEA_TOKEN_PATH - - -def _gitea_token_available() -> bool: - """Check credential readiness without loading it into process memory.""" - try: - path = _gitea_token_path() - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - def sync_cassandra_repo(env_values: dict[str, str]) -> str: - """Clone or fetch Cassandra through the runtime-only askpass credential.""" - token_available = _gitea_token_available() + """Clone or fetch Cassandra through the credential-isolated SCM broker.""" if shutil.which("git") is None: return "git-unavailable" child_env = os.environ.copy() @@ -173,9 +159,7 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: } } ) - child_env["GIT_ASKPASS"] = env_values.get( - "GIT_ASKPASS", "/opt/coordinator/gitea_askpass.sh" - ) + child_env.pop("GIT_ASKPASS", None) child_env["GIT_TERMINAL_PROMPT"] = "0" if (CASSANDRA_BASE_PATH / ".git").exists(): try: @@ -232,8 +216,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: return f"remote-repair-failed-{repaired.returncode}" except (OSError, subprocess.TimeoutExpired): return "remote-repair-failed" - if not token_available: - return "ready; fetch skipped until Gitea token is configured" command = [ "git", "-C", @@ -247,8 +229,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_BASE_PATH.mkdir(parents=True, exist_ok=True) if any(CASSANDRA_BASE_PATH.iterdir()): return "unmanaged-nonempty-directory" - if not token_available: - return "awaiting-gitea-token" command = [ "git", "clone", diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py new file mode 100644 index 00000000..b1795e99 --- /dev/null +++ b/services/hermes/scripts/node_account_audit.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Fail-closed supplementary-group and host privilege-policy audits.""" + +from __future__ import annotations + +import re +import stat +import struct +from pathlib import Path + +from node_account_io import HardeningError, read_regular + + +def _members(field: str, context: str) -> set[str]: + if not field: + return set() + values = field.split(",") + if any(not value or value.strip() != value for value in values): + raise HardeningError(f"{context} has a malformed member list") + if len(values) != len(set(values)): + raise HardeningError(f"{context} has duplicate members") + return set(values) + + +def audit_membership( + account: str, groups: list[list[str]], gshadow: list[list[str]] +) -> None: + """Reject the dedicated account in any supplementary group/admin list.""" + for record in groups: + if account in _members(record[3], f"group {record[0]}"): + raise HardeningError("dedicated Hermes account has supplementary group access") + for record in gshadow: + principals = _members(record[2], f"gshadow {record[0]} admins") + principals |= _members(record[3], f"gshadow {record[0]} members") + if account in principals: + raise HardeningError("dedicated Hermes account has gshadow group access") + + +def _policy_files(host_etc: Path, polkit_share: Path, root_uid: int) -> list[Path]: + files: list[Path] = [] + sudoers = host_etc / "sudoers" + if sudoers.exists(): + files.append(sudoers) + roots = ( + host_etc / "sudoers.d", + host_etc / "polkit-1/rules.d", + host_etc / "polkit-1/localauthority", + polkit_share / "rules.d", + polkit_share / "localauthority", + ) + for root in roots: + if not root.exists(): + continue + metadata = root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != root_uid + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise HardeningError("unsafe sudo/polkit policy directory") + files.extend(path for path in root.rglob("*") if path.is_file()) + if len(files) > 256: + raise HardeningError("too many sudo/polkit policy files") + return sorted(files) + + +def _audit_policy_metadata(snapshot, root_uid: int, account_uid: int) -> None: + if snapshot.uid != root_uid or snapshot.mode & 0o022: + raise HardeningError("sudo/polkit policy metadata permits unsafe mutation") + acl = dict(snapshot.xattrs).get("system.posix_acl_access", b"") + if not acl: + return + header = struct.Struct(" str: + """Remove comments without discarding sudoers' numeric-UID principals.""" + active: list[str] = [] + for line in text.splitlines(): + stripped = line.lstrip() + if not stripped: + continue + include = re.fullmatch( + r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped + ) + if include: + directive, target = include.groups() + if directive.lower().endswith("includedir") and target == "/etc/sudoers.d": + continue + raise HardeningError("sudo policy includes an unaudited authority source") + if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): + continue + active.append(line) + return "\n".join(active) + + +def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> bool: + account_pattern = rf"(? None: + """Reject direct, group, wildcard sudo, or root-equivalent polkit grants.""" + dangerous_polkit = ( + "org.freedesktop.policykit.exec", + "org.freedesktop.systemd1.manage-unit", + "org.freedesktop.udisks2.modify-device", + "org.freedesktop.packagekit", + ) + nsswitch = host_etc / "nsswitch.conf" + if nsswitch.exists(): + try: + nss_snapshot = read_regular(nsswitch, 64 * 1024) + _audit_policy_metadata(nss_snapshot, root_uid, account_uid) + text = nss_snapshot.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("nsswitch policy is not UTF-8") from exc + for line in text.splitlines(): + name, separator, sources = line.partition(":") + if name.strip() in { + "passwd", + "group", + "initgroups", + "shadow", + "sudoers", + } and separator: + active_sources = [item for item in sources.split() if not item.startswith("[")] + if not active_sources or any( + item not in {"files", "systemd"} for item in active_sources + ): + raise HardeningError("external group/account/sudo authority source is unsafe") + total = 0 + for path in _policy_files(host_etc, polkit_share, root_uid): + snapshot = read_regular(path, 256 * 1024) + _audit_policy_metadata(snapshot, root_uid, account_uid) + value = snapshot.value + total += len(value) + if total > 2 * 1024 * 1024: + raise HardeningError("sudo/polkit policy input exceeds safe limit") + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("sudo/polkit policy is not UTF-8") from exc + if path.name == "sudoers" or "sudoers.d" in path.parts: + active = _active_sudo_policy(text) + if _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has sudo authority") + for line in active.splitlines(): + fields = line.split() + if fields and fields[0] in {"ALL", "%ALL"} and "=" in line: + raise HardeningError("broad sudo authority includes Hermes") + continue + active = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + grants = "polkit.Result.YES" in active or re.search( + r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$", active + ) + if grants and _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has polkit authority") + broad = "unix-user:*" in active or "Identity=unix-user:*" in active + if broad and grants and any(item in active for item in dangerous_polkit): + raise HardeningError("root-equivalent broad polkit authority includes Hermes") diff --git a/services/hermes/scripts/node_account_hardening.py b/services/hermes/scripts/node_account_hardening.py new file mode 100644 index 00000000..a545af47 --- /dev/null +++ b/services/hermes/scripts/node_account_hardening.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Reconcile a dedicated unprivileged Hermes SSH account on one Atlas node.""" + +from __future__ import annotations + +import argparse +import base64 +import errno +import os +import re +import shlex +import stat +import struct +from contextlib import suppress +from pathlib import Path + +from node_account_audit import audit_membership, audit_privilege_policies +from node_account_io import ( + FileSnapshot, + HardeningError, + account_lock, + assert_unchanged, + atomic_write, + backup_once, + read_regular, +) + +HOST_ETC = Path("/host-etc") +HOST_HOME = Path("/host-home") +HOST_K3S = Path("/host-k3s") +HOST_KUBELET = Path("/host-kubelet") +HOST_RUN_K3S = Path("/host-run-k3s") +HOST_RUN_CONTAINERD = Path("/host-run-containerd") +HOST_POLKIT_SHARE = Path("/host-polkit-share") +ACCOUNT = "hermes-agent" +ACCOUNT_UID = 1200 +ACCOUNT_GID = 1200 +ACCOUNT_HOME = "/home/hermes-agent" +ACCOUNT_SHELL = "/bin/bash" +HOST_ROOT_UID = 0 +HOST_ROOT_GID = 0 +LEGACY_ACCOUNTS = ("atlas", "oceanus") +MAX_ACCOUNT_FILE = 2 * 1024 * 1024 +MAX_AUTHORIZED_KEYS = 1024 * 1024 +ACL_VERSION = 2 +ACL_UNDEFINED_ID = 0xFFFFFFFF +ACL_USER_OBJ = 0x01 +ACL_USER = 0x02 +ACL_GROUP_OBJ = 0x04 +ACL_GROUP = 0x08 +ACL_MASK = 0x10 +ACL_OTHER = 0x20 +ACL_HEADER = struct.Struct(" tuple[bytes, os.stat_result]: + snapshot = read_regular(path, maximum) + return snapshot.value, path.stat(follow_symlinks=False) + + +def _records(value: bytes, fields: int, name: str) -> list[list[str]]: + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError(f"{name} is not UTF-8") from exc + if not text.endswith("\n"): + raise HardeningError(f"{name} is missing its final newline") + records = [] + names = set() + for line in text.splitlines(): + parts = line.split(":") + if len(parts) != fields or not parts[0] or parts[0] in names: + raise HardeningError(f"{name} has an invalid record") + names.add(parts[0]) + records.append(parts) + return records + + +def _encode(records: list[list[str]]) -> bytes: + return ("\n".join(":".join(record) for record in records) + "\n").encode() + + +def _expected_records() -> dict[str, list[str]]: + return { + "passwd": [ + ACCOUNT, + "x", + str(ACCOUNT_UID), + str(ACCOUNT_GID), + "Hermes Agent", + ACCOUNT_HOME, + ACCOUNT_SHELL, + ], + "group": [ACCOUNT, "x", str(ACCOUNT_GID), ""], + "shadow": [ACCOUNT, "!", "1", "0", "99999", "7", "", "", ""], + "gshadow": [ACCOUNT, "!", "", ""], + } + + +def _reconcile_record( + records: list[list[str]], expected: list[str], *, identity_index: int +) -> list[list[str]]: + for record in records: + same_name = record[0] == expected[0] + same_identity = record[identity_index] == expected[identity_index] + if same_name or same_identity: + if record != expected: + raise HardeningError("dedicated Hermes account identity conflicts") + return records + return [*records, expected] + + +def _reconcile_databases() -> None: + expected = _expected_records() + definitions = ( + ("passwd", 7, 2), + ("group", 4, 2), + ("shadow", 9, 0), + ("gshadow", 4, 0), + ) + planned: list[tuple[Path, FileSnapshot, bytes]] = [] + account_present = False + parsed: dict[str, list[list[str]]] = {} + for name, fields, identity_index in definitions: + path = HOST_ETC / name + snapshot = read_regular(path, MAX_ACCOUNT_FILE) + records = _records(snapshot.value, fields, name) + parsed[name] = records + updated = _reconcile_record( + records, expected[name], identity_index=identity_index + ) + if name == "passwd": + account_present = expected[name] in records + planned.append((path, snapshot, _encode(updated))) + audit_membership(ACCOUNT, parsed["group"], parsed["gshadow"]) + audit_privilege_policies( + ACCOUNT, ACCOUNT_UID, HOST_ROOT_UID, HOST_ETC, HOST_POLKIT_SHARE + ) + try: + (HOST_HOME / ACCOUNT).lstat() + except FileNotFoundError: + pass + else: + if not account_present: + raise HardeningError("dedicated Hermes account home already exists") + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + backup_once(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + written: list[tuple[Path, FileSnapshot, bytes]] = [] + try: + for path, snapshot, updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + atomic_write(path, updated, snapshot) + written.append((path, snapshot, updated)) + for path, _snapshot, _updated in planned: + name = path.name + fields = next(item[1] for item in definitions if item[0] == name) + records = _records(_read_regular(path)[0], fields, name) + if expected[name] not in records: + raise HardeningError("dedicated Hermes account validation failed") + except Exception as error: + for path, snapshot, updated in reversed(written): + current = read_regular(path, MAX_ACCOUNT_FILE) + if ( + current.value != updated + or current.mode != snapshot.mode + or current.uid != snapshot.uid + or current.gid != snapshot.gid + or current.xattrs != snapshot.xattrs + ): + raise HardeningError( + f"concurrent host account change prevents rollback: {path.name}" + ) from error + atomic_write(path, snapshot.value, snapshot) + raise + + +SUPPORTED_KEY_TYPES = {"ssh-ed25519", "ecdsa-sha2-nistp256", "ssh-rsa"} +KEY_TYPE_RE = re.compile(r"(?:sk-)?(?:ssh|ecdsa)-[A-Za-z0-9@._+-]+\Z") + + +def _key_identity(line: bytes) -> tuple[str, bytes] | None: + """Return OpenSSH key type/blob, ignoring options and comments.""" + stripped = line.strip(b"\r\n") + if not stripped or stripped.lstrip().startswith(b"#"): + return None + try: + fields = shlex.split(stripped.decode("ascii"), posix=True) + except (UnicodeDecodeError, ValueError) as exc: + raise HardeningError("authorized key line is malformed") from exc + indexes = [ + index for index, field in enumerate(fields) if KEY_TYPE_RE.fullmatch(field) + ] + if len(indexes) != 1 or indexes[0] + 1 >= len(fields): + raise HardeningError("authorized key line has ambiguous key material") + key_type = fields[indexes[0]] + try: + blob = base64.b64decode(fields[indexes[0] + 1], validate=True) + except ValueError as exc: + raise HardeningError("authorized key payload is invalid") from exc + encoded_type = key_type.encode("ascii") + if len(blob) < 4 or int.from_bytes(blob[:4], "big") != len(encoded_type): + raise HardeningError("authorized key blob is malformed") + if blob[4 : 4 + len(encoded_type)] != encoded_type: + raise HardeningError("authorized key type does not match its blob") + return key_type, blob + + +def _validated_public_key(path: Path) -> tuple[bytes, tuple[str, bytes]]: + value, _ = _read_regular(path, 16 * 1024) + line = value.strip() + if b"\n" in line or b"\r" in line: + raise HardeningError("Hermes public key must contain one line") + identity = _key_identity(line) + fields = line.split() + if ( + identity is None + or identity[0] not in SUPPORTED_KEY_TYPES + or len(fields) not in {2, 3} + or fields[0].decode("ascii") != identity[0] + ): + raise HardeningError("Hermes public key format is unsupported") + return line, identity + + +def _directory(path: Path, *, mode: int, uid: int, gid: int) -> None: + with suppress(FileExistsError): + path.mkdir(mode=mode) + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe account directory: {path.name}") + if metadata.st_uid not in {0, uid} or metadata.st_gid not in {0, gid}: + raise HardeningError(f"account directory ownership conflicts: {path.name}") + os.chown(path, uid, gid) + path.chmod(mode) + + +def _without_key(value: bytes, identity: tuple[str, bytes]) -> bytes: + kept = [] + for line in value.splitlines(keepends=True): + if _key_identity(line) != identity: + kept.append(line) + return b"".join(kept) + + +def _move_key(public_key: Path) -> None: + key, identity = _validated_public_key(public_key) + target = HOST_HOME / ACCOUNT / ".ssh" / "authorized_keys" + legacy_plan: list[tuple[Path, FileSnapshot, bytes]] = [] + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + snapshot = read_regular(authorized, MAX_AUTHORIZED_KEYS) + except FileNotFoundError: + continue + updated = _without_key(snapshot.value, identity) + if updated != snapshot.value: + legacy_plan.append((authorized, snapshot, updated)) + if legacy_plan and target.exists(): + target_snapshot = read_regular(target, MAX_AUTHORIZED_KEYS) + target_without_key = _without_key(target_snapshot.value, identity) + if target_without_key != target_snapshot.value: + backup_once(target, target_snapshot, MAX_AUTHORIZED_KEYS) + assert_unchanged(target, target_snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(target, target_without_key, target_snapshot) + verified_target = read_regular(target, MAX_AUTHORIZED_KEYS).value + if _without_key(verified_target, identity) != verified_target: + raise HardeningError("preexisting Hermes authorization removal failed") + for authorized, snapshot, _updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + backup_once(authorized, snapshot, MAX_AUTHORIZED_KEYS) + for authorized, snapshot, updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(authorized, updated, snapshot) + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + value = read_regular(authorized, MAX_AUTHORIZED_KEYS).value + except FileNotFoundError: + continue + if _without_key(value, identity) != value: + raise HardeningError("legacy Hermes authorization key removal failed") + + # Installation happens only after every legacy authorization is absent. A + # crash can therefore remove access temporarily, but can never duplicate + # the machine credential across privileged and unprivileged accounts. + home = HOST_HOME / ACCOUNT + ssh = home / ".ssh" + _directory(home, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + _directory(ssh, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + target = ssh / "authorized_keys" + if target.exists(): + current = read_regular(target, MAX_AUTHORIZED_KEYS) + if current.value != key + b"\n": + backup_once(target, current, MAX_AUTHORIZED_KEYS) + else: + current = FileSnapshot( + value=b"", + device=0, + inode=0, + mode=0o600, + uid=ACCOUNT_UID, + gid=ACCOUNT_GID, + size=0, + mtime_ns=0, + ctime_ns=0, + xattrs=(), + ) + atomic_write(target, key + b"\n", current) + if _read_regular(target, MAX_AUTHORIZED_KEYS)[0] != key + b"\n": + raise HardeningError("dedicated Hermes authorized key validation failed") + + +def _decode_acl(value: bytes, mode: int) -> list[tuple[int, int, int]]: + """Decode a bounded POSIX ACL or derive one from ordinary mode bits.""" + if not value: + group = (mode >> 3) & 0o7 + return [ + (ACL_USER_OBJ, (mode >> 6) & 0o7, ACL_UNDEFINED_ID), + (ACL_GROUP_OBJ, group, ACL_UNDEFINED_ID), + (ACL_MASK, group, ACL_UNDEFINED_ID), + (ACL_OTHER, mode & 0o7, ACL_UNDEFINED_ID), + ] + if len(value) < ACL_HEADER.size or (len(value) - ACL_HEADER.size) % ACL_ENTRY.size: + raise HardeningError("sensitive directory ACL is malformed") + if ACL_HEADER.unpack_from(value)[0] != ACL_VERSION: + raise HardeningError("sensitive directory ACL version is unsupported") + entries = [ + ACL_ENTRY.unpack_from(value, offset) + for offset in range(ACL_HEADER.size, len(value), ACL_ENTRY.size) + ] + if any(permission > 0o7 for _tag, permission, _identifier in entries): + raise HardeningError("sensitive directory ACL permission is malformed") + required = {ACL_USER_OBJ, ACL_GROUP_OBJ, ACL_OTHER} + if not required <= {tag for tag, _permission, _identifier in entries}: + raise HardeningError("sensitive directory ACL is incomplete") + return entries + + +def _encode_acl(entries: list[tuple[int, int, int]]) -> bytes: + return ACL_HEADER.pack(ACL_VERSION) + b"".join( + ACL_ENTRY.pack(*entry) for entry in entries + ) + + +def _acl_with_deny(value: bytes, mode: int) -> bytes: + entries = _decode_acl(value, mode) + entries = [ + entry + for entry in entries + if not (entry[0] == ACL_USER and entry[2] == ACCOUNT_UID) + ] + entries.append((ACL_USER, 0, ACCOUNT_UID)) + if not any(tag == ACL_MASK for tag, _permission, _identifier in entries): + entries.append((ACL_MASK, (mode >> 3) & 0o7, ACL_UNDEFINED_ID)) + order = { + ACL_USER_OBJ: 0, + ACL_USER: 1, + ACL_GROUP_OBJ: 2, + ACL_GROUP: 3, + ACL_MASK: 4, + ACL_OTHER: 5, + } + entries.sort(key=lambda entry: (order.get(entry[0], 99), entry[2])) + return _encode_acl(entries) + + +def _read_acl(path: Path) -> bytes: + try: + return os.getxattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENODATA, getattr(errno, "ENOATTR", errno.ENODATA)}: + return b"" + raise + + +def _validate_acl_backup(value: bytes) -> None: + if value[:1] not in {b"A", b"N"}: + raise HardeningError("sensitive directory ACL backup is malformed") + if value[:1] == b"A": + if len(value) == 1: + raise HardeningError("sensitive directory ACL backup is malformed") + _decode_acl(value[1:], 0) + elif value != b"N": + raise HardeningError("sensitive directory ACL backup is malformed") + + +def _acl_backup_once(path: Path, value: bytes, backup_name: str | None = None) -> None: + """Persist the original ACL as a private, durable host recovery file.""" + root = HOST_ETC / "hermes-node-boundary" + _directory(root, mode=0o700, uid=HOST_ROOT_UID, gid=HOST_ROOT_GID) + backup = root / f"{backup_name or path.name}.acl" + encoded = b"A" + value if value else b"N" + try: + existing, _metadata = _read_regular(backup, 64 * 1024) + except FileNotFoundError: + pass + else: + _validate_acl_backup(existing) + return + + temporary = root / f".{path.name}.acl.hermes-{os.getpid()}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + if os.write(descriptor, encoded) != len(encoded): + raise HardeningError("short sensitive directory ACL backup write") + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, HOST_ROOT_UID, HOST_ROOT_GID) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + try: + os.link(temporary, backup, follow_symlinks=False) + except FileExistsError: + # Another reconciler won the race; never overwrite the first backup. + pass + finally: + temporary.unlink(missing_ok=True) + stored, _metadata = _read_regular(backup, 64 * 1024) + _validate_acl_backup(stored) + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _deny_sensitive_root(path: Path, backup_name: str | None = None) -> None: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe sensitive directory: {path.name}") + if metadata.st_uid != HOST_ROOT_UID: + raise HardeningError(f"sensitive directory is not root-owned: {path.name}") + current = _read_acl(path) + _acl_backup_once(path, current, backup_name) + updated = _acl_with_deny(current, stat.S_IMODE(metadata.st_mode)) + try: + os.setxattr(path, ACL_XATTR, updated, follow_symlinks=False) + verified = _decode_acl(_read_acl(path), stat.S_IMODE(metadata.st_mode)) + if (ACL_USER, 0, ACCOUNT_UID) not in verified: + raise HardeningError("sensitive directory ACL validation failed") + except Exception: + if current: + os.setxattr(path, ACL_XATTR, current, follow_symlinks=False) + else: + try: + os.removexattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno not in { + errno.ENODATA, + getattr(errno, "ENOATTR", errno.ENODATA), + }: + raise + raise + + +def _deny_sensitive_roots() -> None: + for path, backup_name in ( + (HOST_K3S, "var-lib-rancher-k3s"), + (HOST_KUBELET, "var-lib-kubelet"), + (HOST_RUN_K3S, "run-k3s"), + (HOST_RUN_CONTAINERD, "run-containerd"), + ): + _deny_sensitive_root(path, backup_name) + + +def reconcile(public_key: Path) -> None: + """Create the locked account and move only the Hermes authorization key.""" + with account_lock(HOST_ETC, expected_uid=HOST_ROOT_UID): + _reconcile_databases() + _deny_sensitive_roots() + _move_key(public_key) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--public-key-file", type=Path, required=True) + args = parser.parse_args() + reconcile(args.public_key_file) + print("Dedicated Hermes node account reconciled.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_io.py b/services/hermes/scripts/node_account_io.py new file mode 100644 index 00000000..4fb2ae9e --- /dev/null +++ b/services/hermes/scripts/node_account_io.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Crash-safe, metadata-preserving host account database I/O.""" + +from __future__ import annotations + +import fcntl +import os +import stat +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +MAX_XATTRS = 64 +MAX_XATTR_NAME = 255 +MAX_XATTR_VALUE = 256 * 1024 +MAX_XATTR_TOTAL = 1024 * 1024 +LOCK_TIMEOUT_SECONDS = 15.0 + + +class HardeningError(RuntimeError): + """Raised before an unsafe or ambiguous host-account change.""" + + +@dataclass(frozen=True) +class FileSnapshot: + """Bounded file contents and all security-relevant inode metadata.""" + + value: bytes + device: int + inode: int + mode: int + uid: int + gid: int + size: int + mtime_ns: int + ctime_ns: int + xattrs: tuple[tuple[str, bytes], ...] + + +def _stat_identity(item: os.stat_result) -> tuple[int, ...]: + return ( + item.st_dev, + item.st_ino, + item.st_mode, + item.st_uid, + item.st_gid, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + +def _xattrs(path_or_fd: Path | int) -> tuple[tuple[str, bytes], ...]: + try: + names = sorted(os.listxattr(path_or_fd, follow_symlinks=False)) + except (TypeError, ValueError): + names = sorted(os.listxattr(path_or_fd)) + if len(names) > MAX_XATTRS: + raise HardeningError("host account file has too many extended attributes") + values: list[tuple[str, bytes]] = [] + total = 0 + for name in names: + if len(name.encode("utf-8")) > MAX_XATTR_NAME: + raise HardeningError("host account file xattr name is too long") + try: + value = os.getxattr(path_or_fd, name, follow_symlinks=False) + except (TypeError, ValueError): + value = os.getxattr(path_or_fd, name) + total += len(value) + if len(value) > MAX_XATTR_VALUE or total > MAX_XATTR_TOTAL: + raise HardeningError("host account file xattrs exceed the safe limit") + values.append((name, value)) + return tuple(values) + + +def _restore_xattrs(descriptor: int, expected: tuple[tuple[str, bytes], ...]) -> None: + expected_names = {name for name, _value in expected} + for name, _value in _xattrs(descriptor): + if name not in expected_names: + os.removexattr(descriptor, name) + for name, value in expected: + os.setxattr(descriptor, name, value) + + +def read_regular(path: Path, maximum: int) -> FileSnapshot: + """Read one bounded, non-symlink regular file and capture its metadata.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum: + raise HardeningError(f"unsafe regular file: {path.name}") + value = os.read(descriptor, maximum + 1) + attrs = _xattrs(descriptor) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + if len(value) != metadata.st_size or _stat_identity(metadata) != _stat_identity(after): + raise HardeningError(f"short file read: {path.name}") + return FileSnapshot( + value=value, + device=metadata.st_dev, + inode=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + uid=metadata.st_uid, + gid=metadata.st_gid, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + xattrs=attrs, + ) + + +def assert_unchanged(path: Path, expected: FileSnapshot, maximum: int) -> None: + """Fail if an account file changed since planning began.""" + current = read_regular(path, maximum) + if current != expected: + raise HardeningError(f"concurrent host account change detected: {path.name}") + + +def atomic_write(path: Path, value: bytes, metadata: FileSnapshot) -> None: + """Atomically replace a file while retaining ACLs, labels, and xattrs.""" + temporary = path.with_name(f".{path.name}.hermes-{os.getpid()}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, metadata.mode) + try: + if os.write(descriptor, value) != len(value): + raise HardeningError(f"short atomic write: {path.name}") + os.fchmod(descriptor, metadata.mode) + os.fchown(descriptor, metadata.uid, metadata.gid) + _restore_xattrs(descriptor, metadata.xattrs) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def backup_once(path: Path, snapshot: FileSnapshot, maximum: int) -> Path: + """Create the first durable recovery copy without replacing an older one.""" + backup = path.with_name(path.name + ".hermes-boundary-backup") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(backup, flags, snapshot.mode) + except FileExistsError: + read_regular(backup, maximum) + return backup + try: + if os.write(descriptor, snapshot.value) != len(snapshot.value): + raise HardeningError(f"short backup write: {path.name}") + os.fchmod(descriptor, snapshot.mode) + os.fchown(descriptor, snapshot.uid, snapshot.gid) + _restore_xattrs(descriptor, snapshot.xattrs) + os.fsync(descriptor) + except Exception: + backup.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + return backup + + +@contextmanager +def account_lock(host_etc: Path, *, expected_uid: int = 0) -> Iterator[None]: + """Hold the standard shadow-utils account lock for the full transaction.""" + path = host_etc / ".pwd.lock" + flags = os.O_WRONLY | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != expected_uid + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise HardeningError("unsafe standard account lock") + deadline = time.monotonic() + LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise HardeningError("standard account lock is busy") from exc + time.sleep(0.05) + yield + finally: + try: + fcntl.lockf(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..07423d70 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -86,6 +86,21 @@ def _write_empty_auth_store() -> None: os.chown(path, OWNER_UID, OWNER_GID) +def _stage_node_ssh_config() -> None: + """Force every Atlas node alias onto the dedicated Hermes OS account.""" + destination = RUNTIME_ROOT / "node-ssh-config" + source = _copy_secret("node-ssh-config", destination) + if "\x00" in source: + destination.unlink(missing_ok=True) + raise RuntimeError("node SSH config contains invalid control data") + destination.write_text( + "Host titan-*\n User hermes-agent\n" + source.rstrip() + "\n", + encoding="utf-8", + ) + destination.chmod(0o600) + os.chown(destination, OWNER_UID, OWNER_GID) + + def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): @@ -93,13 +108,11 @@ def stage_agent() -> None: for name in ( "agent-api-key", "chat-relay-key", - "gitea-token", - "gitea-username", "node-ssh-private-key", - "node-ssh-config", "node-ssh-known-hosts", ): _copy_secret(name, RUNTIME_ROOT / name) + _stage_node_ssh_config() _validated_json( "claude-credentials", RUNTIME_ROOT / "claude" / ".credentials.json", diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 32fd1ccf..6194f7f9 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -5,17 +5,21 @@ description: Read bounded private Atlas repository and pull-request metadata or # Manage Atlas pull requests -Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token -from the pod-lifetime Vault projection; never read that file, copy the token, -put it in an argument or environment variable, or replace this client with -`curl`. +Use `/opt/scm/gitea_api.py` for Forgejo API access. The agent pod has no +repository credential. The client calls a separate least-authority broker; +never bypass it with `curl` or direct Gitea HTTP. + +Use the configured broker Git remote for clone, fetch, and creation of a new +namespaced feature branch. Existing-ref updates, protected refs, deletion, +and force-push are rejected by the broker. Never replace that remote with a +credential-bearing URL. ## Read repository or PR state Pass one allowed Atlas metadata API path to the read operation: ```sh -/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +/opt/scm/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` The client exposes only repository metadata, PRs and PR evidence, branches, @@ -29,10 +33,16 @@ evidence, not authorization to change it. Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and exact pushed commit. Never force-push. Pass that full 40-character pushed head SHA explicitly. Keep the short body free of credentials and credential-like -text. Validate without reading a credential or using the network: +text. The client screens structured assignments, known token formats, and long +high-entropy values as an accidental-disclosure barrier. It cannot prove text +is secret-free: short or multiword secrets under innocuous names may resemble +ordinary prose, so never place credential material in either field. Structured +inspection is deliberately bounded; large, deeply nested, or unusual config +blobs do not belong in these fields and may be rejected. Validate without +reading a credential or using the network: ```sh -/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ +/opt/scm/gitea_api.py --dry-run create-draft REPO \ --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ --title "Focused change" --body "Tests and review evidence" ``` diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..7804a9f3 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-9 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..612391c9 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -256,7 +256,9 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ - "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" + "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \ + "hermes/developer-gitea" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..805857d9 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -20,6 +20,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/__init__.py", "testing/quality_contract.py", "testing/quality_docs.py", @@ -43,6 +55,13 @@ "services/comms/scripts/tests", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/tests", "testing" ], @@ -58,6 +77,9 @@ "coverage_sources": [ "ci/scripts", "scripts.render.dashboards_render_atlas", + "services/gitea/scripts", + "services/hermes/scm-common/scripts", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -124,6 +146,13 @@ "ci/tests/**/*.py", "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/*.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py" ], @@ -167,6 +196,22 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "testing/quality_coverage.py" + ], "tracked_files": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", @@ -174,6 +219,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/quality_contract.py", "testing/quality_docs.py", "testing/quality_hygiene.py", diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..98cc65d5 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -47,17 +58,39 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] percentages = _load_percentages(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + branch_minimum = coverage_contract.get("minimum_branch_percent") + if branch_minimum is not None: + branch_minimum = float(branch_minimum) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get( + "branch_tracked_files", + coverage_contract.get("tracked_files", []) + if branch_minimum is not None + else [], + ) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if branch_minimum is None or normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + "branch coverage below " + f"{branch_minimum:.1f}%: {relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +110,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py new file mode 100644 index 00000000..48c6c2e5 --- /dev/null +++ b/testing/tests/test_hermes_chat_images.py @@ -0,0 +1,282 @@ +"""Image generation and sandbox-boundary contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import sys +import time +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( + HERMES, + VAULT, + _documents, +) + + +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert ( + "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + ) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any( + mount["name"] == "home" and "agent" in str(mount) + for mount in hermes["volumeMounts"] + ) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + agent_role = vault_policy[ + vault_policy.index('write_policy_and_role "hermes-agent"') : vault_policy.index( + "write_policy_and_role", + vault_policy.index('write_policy_and_role "hermes-agent"') + 1, + ) + ] + assert "hermes/developer-gitea" not in agent_role + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault_policy + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) + + +def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): + """Follow-up edits resolve the source server-side and keep tool JSON small.""" + provider_module = SimpleNamespace( + DEFAULT_ASPECT_RATIO="square", + ImageGenProvider=object, + error_response=lambda **value: value, + normalize_reference_images=lambda value: value, + resolve_aspect_ratio=lambda value: value, + save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", + success_response=lambda **value: value, + ) + monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) + spec = importlib.util.spec_from_file_location( + "hermes_image_plugin", + HERMES / "plugins" / "image-gen-broker" / "__init__.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + image_dir = tmp_path / "cache" / "images" + image_dir.mkdir(parents=True) + older = image_dir / "atlas_flux-old.png" + newest = image_dir / "atlas_gpt-image-new.png" + older.write_bytes(b"older") + newest.write_bytes(b"newest") + older.touch() + time.sleep(0.001) + newest.touch() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + calls = [] + monkeypatch.setattr( + module, + "_handle_image_generate", + lambda args, route: calls.append((args, route)) or "ok", + ) + assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" + assert calls == [ + ( + { + "prompt": "make it a clown", + "image_url": str(newest.resolve()), + }, + "hosted", + ) + ] + + +def test_chat_reasoning_uses_switchyard_without_owner_credentials(): + """Family pods use AUTO/manual routes without mounting owner credentials.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/fast", + "model": "atlas/auto/fast", + } + assert config["providers"]["atlas-switchyard"] == { + "name": "Automatic Router", + "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", + "api_key": "atlas-switchyard", + "default_model": "atlas/auto/fast", + "transport": "chat_completions", + } + assert config["platforms"]["api_server"]["extra"]["model_routes"] == { + route: {"provider": "atlas-switchyard", "model": route} + for route in [ + "atlas/auto/fast", + "atlas/auto/balanced", + "atlas/auto/deep", + "atlas/auto/maximum", + "atlas/manual/codex/luna", + "atlas/manual/codex/terra", + "atlas/manual/codex/sol", + "atlas/manual/claude/haiku", + "atlas/manual/claude/fable", + "atlas/manual/claude/sonnet", + "atlas/manual/claude/opus", + "atlas/manual/local/qwen-14b", + ] + } + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "codex-broker") + assert broker["ports"] == [ + {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { + "PYTHONPATH": "/opt/hermes", + "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", + "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", + }.items() + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-codex-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + assert service["spec"]["ports"] == [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP", + } + ] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + assert ( + statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] + == "20260816-telegram-topics" + ) + pod_spec = statefulset["spec"]["template"]["spec"] + patch_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-stream-recovery" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/conversation_loop.py", + "/patched/conversation_loop.py", + ] + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") + assert { + "name": "stream-recovery-patch", + "mountPath": "/opt/hermes/agent/conversation_loop.py", + "subPath": "conversation_loop.py", + } in hermes["volumeMounts"] + api_session_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + assert "patch_api_server_sessions.py" in api_session_init["args"][0] + assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] + assert { + "name": "api-server-patch", + "mountPath": "/opt/hermes/gateway/platforms/api_server.py", + "subPath": "api_server.py", + } in hermes["volumeMounts"] + assert any(volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]) + assert not any( + mount["mountPath"].endswith("/.codex") for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py new file mode 100644 index 00000000..ddb0ef40 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -0,0 +1,172 @@ +"""Local inference and sandbox execution contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") + + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + "\n".join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert "OFFLOAD_MODE = os.environ.get(" in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item + for item in container["volumeMounts"] + if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith( + "/chrome-linux/headless_shell" + ) + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py new file mode 100644 index 00000000..8e432131 --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -0,0 +1,476 @@ +"""Provider subscription and OAuth contracts for Hermes chat.""" + +from __future__ import annotations + +import base64 +import json +import sys +import time +from pathlib import Path +from types import ModuleType + +import pytest + +from testing.tests.test_hermes_chat_support import ( + _load_broker_module, +) + + +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module("hermes_codex_broker", "codex_broker.py", monkeypatch) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload({"model": "gpt-5.6-terra", "input": response_item})[ + "input" + ] == [response_item] + response_items = [response_item] + assert ( + module._validate_payload({"model": "gpt-5.6-terra", "input": response_items})[ + "input" + ] + is response_items + ) + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload({"model": "gpt-5.6-terra", "input": image_items})[ + "input" + ][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"]["output"] == [ + completed_item + ] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert ( + json.loads(normalized_terminal.removeprefix("data: "))["response"]["output"] + == [] + ) + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = ( + base64.urlsafe_b64encode(json.dumps({"exp": time.time() + 3600}).encode()) + .decode() + .rstrip("=") + ) + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = ( + base64.urlsafe_b64encode(json.dumps({"exp": expires_at}).encode()) + .decode() + .rstrip("=") + ) + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 025f3bf7..b513b452 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1,49 +1,18 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" +"""Core configuration and workload contracts for Hermes chat.""" from __future__ import annotations -import base64 import importlib.util -import json -import sqlite3 import sys -import time -import tomllib -from pathlib import Path -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace -import pytest import yaml - -ROOT = Path(__file__).parents[2] -HERMES = ROOT / "services" / "hermes" -VAULT = ROOT / "services" / "vault" - - -def _documents(path: Path) -> list[dict]: - return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] - - -def _load_broker_module(name: str, filename: str, monkeypatch): - """Load one broker with its mounted routing-catalog dependency.""" - catalog_path = HERMES / "scripts" / "routing_catalog.py" - catalog_spec = importlib.util.spec_from_file_location( - "routing_catalog", catalog_path - ) - assert catalog_spec and catalog_spec.loader - catalog = importlib.util.module_from_spec(catalog_spec) - catalog_spec.loader.exec_module(catalog) - monkeypatch.setitem(sys.modules, "routing_catalog", catalog) - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - - broker_path = HERMES / "scripts" / filename - spec = importlib.util.spec_from_file_location(name, broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] @@ -92,13 +61,14 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "brokered Git for clone, fetch" in instructions + assert "client carries no repository credential" in instructions assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul + assert "no repository token is present in this pod" in soul assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions @@ -484,1765 +454,3 @@ def test_voice_workloads_have_deliberate_xavier_placement(): assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" assert all("hostPath" not in volume for volume in stt["volumes"]) assert all("hostPath" not in volume for volume in tts["volumes"]) - - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) - - -def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): - """Follow-up edits resolve the source server-side and keep tool JSON small.""" - provider_module = SimpleNamespace( - DEFAULT_ASPECT_RATIO="square", - ImageGenProvider=object, - error_response=lambda **value: value, - normalize_reference_images=lambda value: value, - resolve_aspect_ratio=lambda value: value, - save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", - success_response=lambda **value: value, - ) - monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) - monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) - spec = importlib.util.spec_from_file_location( - "hermes_image_plugin", - HERMES / "plugins" / "image-gen-broker" / "__init__.py", - ) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - image_dir = tmp_path / "cache" / "images" - image_dir.mkdir(parents=True) - older = image_dir / "atlas_flux-old.png" - newest = image_dir / "atlas_gpt-image-new.png" - older.write_bytes(b"older") - newest.write_bytes(b"newest") - older.touch() - time.sleep(0.001) - newest.touch() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - calls = [] - monkeypatch.setattr( - module, - "_handle_image_generate", - lambda args, route: calls.append((args, route)) or "ok", - ) - assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" - assert calls == [ - ( - { - "prompt": "make it a clown", - "image_url": str(newest.resolve()), - }, - "hosted", - ) - ] - - -def test_chat_reasoning_uses_switchyard_without_owner_credentials(): - """Family pods use AUTO/manual routes without mounting owner credentials.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/fast", - "model": "atlas/auto/fast", - } - assert config["providers"]["atlas-switchyard"] == { - "name": "Automatic Router", - "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", - "api_key": "atlas-switchyard", - "default_model": "atlas/auto/fast", - "transport": "chat_completions", - } - assert config["platforms"]["api_server"]["extra"]["model_routes"] == { - route: {"provider": "atlas-switchyard", "model": route} - for route in [ - "atlas/auto/fast", - "atlas/auto/balanced", - "atlas/auto/deep", - "atlas/auto/maximum", - "atlas/manual/codex/luna", - "atlas/manual/codex/terra", - "atlas/manual/codex/sol", - "atlas/manual/claude/haiku", - "atlas/manual/claude/fable", - "atlas/manual/claude/sonnet", - "atlas/manual/claude/opus", - "atlas/manual/local/qwen-14b", - ] - } - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "codex-broker") - assert broker["ports"] == [ - {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { - "PYTHONPATH": "/opt/hermes", - "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", - "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", - }.items() - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-codex-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - assert service["spec"]["ports"] == [ - { - "name": "http", - "port": 9003, - "targetPort": "codex-broker", - "protocol": "TCP", - } - ] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" - pod_spec = statefulset["spec"]["template"]["spec"] - patch_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-stream-recovery" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/conversation_loop.py", - "/patched/conversation_loop.py", - ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) - assert { - "name": "stream-recovery-patch", - "mountPath": "/opt/hermes/agent/conversation_loop.py", - "subPath": "conversation_loop.py", - } in hermes["volumeMounts"] - api_session_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-api-server-sessions" - ) - assert "patch_api_server_sessions.py" in api_session_init["args"][0] - assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] - assert { - "name": "api-server-patch", - "mountPath": "/opt/hermes/gateway/platforms/api_server.py", - "subPath": "api_server.py", - } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) - assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - - -def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): - """Only named Telegram conversations receive presentation metadata.""" - module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - state = tmp_path / "state.db" - responses = tmp_path / "response_store.db" - with sqlite3.connect(state) as connection: - connection.execute( - """CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, - display_name TEXT, origin_json TEXT, title TEXT - )""" - ) - connection.executemany( - "INSERT INTO sessions (id, source) VALUES (?, ?)", - (("telegram-session", "api_server"), ("other-session", "api_server")), - ) - with sqlite3.connect(responses) as connection: - connection.execute( - "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" - ) - connection.execute( - "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" - ) - connection.executemany( - "INSERT INTO responses VALUES (?, ?, 0)", - ( - ("telegram-response", json.dumps({"session_id": "telegram-session"})), - ("other-response", json.dumps({"session_id": "other-session"})), - ), - ) - connection.executemany( - "INSERT INTO conversations VALUES (?, ?)", - (("telegram", "telegram-response"), ("unrelated", "other-response")), - ) - - assert module.migrate(state, responses) == 1 - assert module.migrate(state, responses) == 0 - with sqlite3.connect(state) as connection: - telegram = connection.execute( - "SELECT session_key, chat_type, display_name, origin_json, title " - "FROM sessions WHERE id = 'telegram-session'" - ).fetchone() - other = connection.execute( - "SELECT session_key, title FROM sessions WHERE id = 'other-session'" - ).fetchone() - assert telegram[:3] == ("telegram", "private", "Telegram") - assert json.loads(telegram[3]) == { - "platform": "telegram", - "session_key": "telegram", - } - assert telegram[4] == "Telegram · General" - assert other == (None, None) - - -def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): - """The DOM transcript includes events without polluting agent history.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "web_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.LATEST_ROWS_BEFORE - + module.LATEST_SELECTION_BEFORE - + module.HELPER_MARKER - + " db = object()\n" - + module.MESSAGES_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "def _run_activity_messages(" in patched - assert 'root = get_hermes_home() / "run-activity"' in patched - assert "path.stat().st_size > 600_000" in patched - assert "entries[-1_000:]" in patched - assert "*_run_activity_messages(sid)" in patched - assert "messages.sort(" in patched - assert "limit: Optional[int] = None" in patched - assert "total_messages = len(messages)" in patched - assert "min(int(limit), 10_000)" in patched - assert '"total_messages": total_messages' in patched - assert "SELECT id, parent_session_id, started_at, ended_at" in patched - assert "newest still-open member of the lineage" in patched - assert "immediate objective parent" in patched - assert "mixes unrelated workstreams" in patched - assert "orphaned children" in patched - assert 'item[0].get("ended_at") is None' in patched - - -def test_web_session_lineage_returns_to_resumed_parent(): - """An ended reviewer must not strand the live view away from its parent.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - namespace: dict[str, object] = {} - exec( - "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, - namespace, - ) - - class FakeDB: - def get_session(self, session_id): - return next((row for row in rows if row["id"] == session_id), None) - - rows = [ - { - "id": "umbrella", - "parent_session_id": None, - "started_at": 0.0, - "ended_at": None, - }, - { - "id": "root", - "parent_session_id": "umbrella", - "started_at": 1.0, - "ended_at": None, - }, - { - "id": "review-1", - "parent_session_id": "root", - "started_at": 2.0, - "ended_at": 3.0, - }, - { - "id": "review-2", - "parent_session_id": "root", - "started_at": 4.0, - "ended_at": None, - }, - ] - select_active = namespace["select_active"] - - assert select_active("root", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - assert select_active("review-1", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - rows[2]["ended_at"] = 5.0 - rows[3]["ended_at"] = 5.0 - assert select_active("root", FakeDB(), rows) == ("root", ["root"]) - - rows[1]["ended_at"] = 9.0 - rows.append( - { - "id": "orphaned-review", - "parent_session_id": "root", - "started_at": 6.0, - "ended_at": None, - } - ) - rows.append( - { - "id": "accepted-review", - "parent_session_id": "root", - "started_at": 7.0, - "ended_at": 8.0, - } - ) - assert select_active("review-1", FakeDB(), rows) == ( - "accepted-review", - ["root", "accepted-review"], - ) - - -def test_api_activity_patch_coalesces_streaming_heartbeats( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """Token callbacks stay bounded while every tool transition is retained.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( - " def _make_run_event_callback(", 1 - )[0] - namespace: dict[str, object] = {} - exec( - "import hashlib, json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, - namespace, - ) - recorder = namespace["ActivityRecorder"]() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - recorder._record_run_activity("session", "_thinking") - recorder._record_run_activity("session", "reasoning.available") - recorder._record_run_activity("session", "subagent.thinking") - recorder._record_run_activity("session", "tool.started", tool_name="terminal") - recorder._record_run_activity("session", "tool.completed", tool_name="terminal") - - journals = list((tmp_path / "run-activity").glob("*.jsonl")) - assert len(journals) == 1 - entries = [json.loads(line) for line in journals[0].read_text().splitlines()] - assert [entry["activity_event"] for entry in entries] == [ - "_thinking", - "tool.started", - "tool.completed", - ] - - -def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): - """Known standalone API workers move under Cassandra without data loss.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", - (module.LEGACY_CASSANDRA_PARENT,), - ) - worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", - (worker_id,), - ) - connection.executemany( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", - ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ) - - assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - row = connection.execute( - "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", - (worker_id,), - ).fetchone() - assert row == ( - module.LEGACY_CASSANDRA_PARENT, - module.LEGACY_CASSANDRA_WORKERS[worker_id], - "keep-me", - ) - with sqlite3.connect(database) as connection: - orphans = connection.execute( - "SELECT id, archived, title, transcript FROM sessions " - "WHERE id IN ({}) ORDER BY id".format( - ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) - ), - tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ).fetchall() - assert orphans == sorted( - ( - orphan_id, - 1, - module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], - "keep-smoke", - ) - for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS - ) - - -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): - """Only the stable Ariadne contract moves below the triage parent.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" - ) - connection.executemany( - "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", - (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), - ) - connection.executemany( - "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", - ( - ( - "triage-run", - module.TRIAGE_MESSAGE_PREFIXES[0] - + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", - ), - ( - "jenkins-run", - module.TRIAGE_MESSAGE_PREFIXES[1] - + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", - ), - ("interactive-run", "Please explain this alert to me."), - ), - ) - - assert module.migrate(database, group_triage=True) == 2 - assert module.migrate(database, group_triage=True) == 0 - with sqlite3.connect(database) as connection: - parent = connection.execute( - "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) - ).fetchone() - triage = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" - ).fetchone() - interactive = connection.execute( - "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" - ).fetchone() - jenkins = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" - ).fetchone() - assert parent == (module.TRIAGE_PARENT_TITLE,) - assert triage == ( - module.TRIAGE_PARENT, - "Sonar · bstein_home · python:S2208 · iage-run", - ) - assert interactive == (None,) - assert jenkins == ( - module.TRIAGE_PARENT, - "Sonar · soteria/291 · kins-run", - ) - - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) - - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) - - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" - - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py new file mode 100644 index 00000000..6a81d625 --- /dev/null +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -0,0 +1,486 @@ +"""Routing and runtime workload contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +import tomllib +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "close_stale_api_workers", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" + ).read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert ( + environment["HERMES_ROUTING_CATALOG_PATH"] + == "/routing-catalog/catalog.json" + ) + assert ( + environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + assert ( + environment["HERMES_CLAUDE_HEALTH_PATH"] + == "/opt/data/provider-health/claude.json" + ) + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert ( + codex_environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) + + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location( + "hermes_classifier_broker", broker_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,private"}, + }, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" + + +def test_image_broker_returns_bytes_and_removes_owner_cache( + tmp_path: Path, monkeypatch +): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible( + tmp_path: Path, monkeypatch +): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert ( + module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) + is None + ) + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert ( + "nvidia-process-exporter-local.monitoring.svc.cluster.local" + in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ( + "configmap.yaml", + "agent-configmap.yaml", + "chat-configmap.yaml", + ): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert ( + "Repeated quality misses require xhigh" + in routes["worker_auto_maximum"]["prompt"] + ) + assert any( + target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate diff --git a/testing/tests/test_hermes_chat_session_continuity.py b/testing/tests/test_hermes_chat_session_continuity.py new file mode 100644 index 00000000..0da542cb --- /dev/null +++ b/testing/tests/test_hermes_chat_session_continuity.py @@ -0,0 +1,478 @@ +"""Session lineage and continuity contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_chat_support import ( + HERMES, +) + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert "detail = tool_name if event_type in {" in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert "heartbeats.get(session_id, 0.0)" in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): + """Only named Telegram conversations receive presentation metadata.""" + module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + spec = importlib.util.spec_from_file_location( + "migrate_telegram_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions (id, source) VALUES (?, ?)", + (("telegram-session", "api_server"), ("other-session", "api_server")), + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("telegram-response", json.dumps({"session_id": "telegram-session"})), + ("other-response", json.dumps({"session_id": "other-session"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "telegram-response"), ("unrelated", "other-response")), + ) + + assert module.migrate(state, responses) == 1 + assert module.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + telegram = connection.execute( + "SELECT session_key, chat_type, display_name, origin_json, title " + "FROM sessions WHERE id = 'telegram-session'" + ).fetchone() + other = connection.execute( + "SELECT session_key, title FROM sessions WHERE id = 'other-session'" + ).fetchone() + assert telegram[:3] == ("telegram", "private", "Telegram") + assert json.loads(telegram[3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert telegram[4] == "Telegram · General" + assert other == (None, None) + + +def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): + """The DOM transcript includes events without polluting agent history.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "web_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.LATEST_ROWS_BEFORE + + module.LATEST_SELECTION_BEFORE + + module.HELPER_MARKER + + " db = object()\n" + + module.MESSAGES_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "def _run_activity_messages(" in patched + assert 'root = get_hermes_home() / "run-activity"' in patched + assert "path.stat().st_size > 600_000" in patched + assert "entries[-1_000:]" in patched + assert "*_run_activity_messages(sid)" in patched + assert "messages.sort(" in patched + assert "limit: Optional[int] = None" in patched + assert "total_messages = len(messages)" in patched + assert "min(int(limit), 10_000)" in patched + assert '"total_messages": total_messages' in patched + assert "SELECT id, parent_session_id, started_at, ended_at" in patched + assert "newest still-open member of the lineage" in patched + assert "immediate objective parent" in patched + assert "mixes unrelated workstreams" in patched + assert "orphaned children" in patched + assert 'item[0].get("ended_at") is None' in patched + + +def test_web_session_lineage_returns_to_resumed_parent(): + """An ended reviewer must not strand the live view away from its parent.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + namespace: dict[str, object] = {} + exec( + "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, + namespace, + ) + + class FakeDB: + def get_session(self, session_id): + return next((row for row in rows if row["id"] == session_id), None) + + rows = [ + { + "id": "umbrella", + "parent_session_id": None, + "started_at": 0.0, + "ended_at": None, + }, + { + "id": "root", + "parent_session_id": "umbrella", + "started_at": 1.0, + "ended_at": None, + }, + { + "id": "review-1", + "parent_session_id": "root", + "started_at": 2.0, + "ended_at": 3.0, + }, + { + "id": "review-2", + "parent_session_id": "root", + "started_at": 4.0, + "ended_at": None, + }, + ] + select_active = namespace["select_active"] + + assert select_active("root", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + assert select_active("review-1", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + rows[2]["ended_at"] = 5.0 + rows[3]["ended_at"] = 5.0 + assert select_active("root", FakeDB(), rows) == ("root", ["root"]) + + rows[1]["ended_at"] = 9.0 + rows.append( + { + "id": "orphaned-review", + "parent_session_id": "root", + "started_at": 6.0, + "ended_at": None, + } + ) + rows.append( + { + "id": "accepted-review", + "parent_session_id": "root", + "started_at": 7.0, + "ended_at": 8.0, + } + ) + assert select_active("review-1", FakeDB(), rows) == ( + "accepted-review", + ["root", "accepted-review"], + ) + + +def test_api_activity_patch_coalesces_streaming_heartbeats( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Token callbacks stay bounded while every tool transition is retained.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( + " def _make_run_event_callback(", 1 + )[0] + namespace: dict[str, object] = {} + exec( + "import hashlib, json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def redact_sensitive_text(value): return value\n" + "class ActivityRecorder:\n" + activity_body, + namespace, + ) + recorder = namespace["ActivityRecorder"]() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + recorder._record_run_activity("session", "_thinking") + recorder._record_run_activity("session", "reasoning.available") + recorder._record_run_activity("session", "subagent.thinking") + recorder._record_run_activity("session", "tool.started", tool_name="terminal") + recorder._record_run_activity("session", "tool.completed", tool_name="terminal") + + journals = list((tmp_path / "run-activity").glob("*.jsonl")) + assert len(journals) == 1 + entries = [json.loads(line) for line in journals[0].read_text().splitlines()] + assert [entry["activity_event"] for entry in entries] == [ + "_thinking", + "tool.started", + "tool.completed", + ] + + +def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): + """Known standalone API workers move under Cassandra without data loss.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", + (module.LEGACY_CASSANDRA_PARENT,), + ) + worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", + (worker_id,), + ) + connection.executemany( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", + ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ) + + assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + row = connection.execute( + "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", + (worker_id,), + ).fetchone() + assert row == ( + module.LEGACY_CASSANDRA_PARENT, + module.LEGACY_CASSANDRA_WORKERS[worker_id], + "keep-me", + ) + with sqlite3.connect(database) as connection: + orphans = connection.execute( + "SELECT id, archived, title, transcript FROM sessions " + "WHERE id IN ({}) ORDER BY id".format( + ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) + ), + tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ).fetchall() + assert orphans == sorted( + ( + orphan_id, + 1, + module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], + "keep-smoke", + ) + for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS + ) + + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs( + tmp_path: Path, +): + """Only the stable Ariadne contract moves below the triage parent.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "migrate_triage_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" + ) + connection.executemany( + "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", + (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), + ) + connection.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", + ( + ( + "triage-run", + module.TRIAGE_MESSAGE_PREFIXES[0] + + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", + ), + ( + "jenkins-run", + module.TRIAGE_MESSAGE_PREFIXES[1] + + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", + ), + ("interactive-run", "Please explain this alert to me."), + ), + ) + + assert module.migrate(database, group_triage=True) == 2 + assert module.migrate(database, group_triage=True) == 0 + with sqlite3.connect(database) as connection: + parent = connection.execute( + "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) + ).fetchone() + triage = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" + ).fetchone() + interactive = connection.execute( + "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" + ).fetchone() + jenkins = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" + ).fetchone() + assert parent == (module.TRIAGE_PARENT_TITLE,) + assert triage == ( + module.TRIAGE_PARENT, + "Sonar · bstein_home · python:S2208 · iage-run", + ) + assert interactive == (None,) + assert jenkins == ( + module.TRIAGE_PARENT, + "Sonar · soteria/291 · kins-run", + ) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py new file mode 100644 index 00000000..bf7d998c --- /dev/null +++ b/testing/tests/test_hermes_chat_support.py @@ -0,0 +1,38 @@ +"""Shared paths and loaders for Hermes chat capability contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import yaml + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" +VAULT = ROOT / "services" / "vault" + + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + + +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..c40740aa 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -1,93 +1,19 @@ -"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" +"""Provider routing and process-isolation contracts for Hermes CLI lanes.""" from __future__ import annotations -import importlib.util import json import os import signal import sys -from contextlib import nullcontext from pathlib import Path -from types import SimpleNamespace - -import pytest -import yaml -SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" -sys.path.insert(0, str(SCRIPTS)) -HERMES = Path(__file__).parents[2] / "services/hermes" -KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" -FLUX_HERMES = ( - Path(__file__).parents[2] - / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" +from testing.tests.test_hermes_cli_lanes_support import ( + _SwitchyardResponse, + lanes, ) - -def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -lanes = _load("cli_lane_runner") -policy = _load("claude_command_policy") -migration = _load("migrate_herdr_state") -auth_patch = _load("patch_hermes_auth") -tui_gateway_patch = _load("patch_tui_gateway") -codex_runtime_patch = _load("patch_codex_runtime") -ttyd_patch = _load("patch_ttyd_index") -client_config = _load("configure_agent_clients") - - -def _agent_deployment() -> dict: - return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) - - -def _services() -> dict[str, dict]: - return { - item["metadata"]["name"]: item - for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) - if item - } - - -def _oauth_deployment(name: str) -> dict: - documents = [ - item - for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) - if item - ] - return next( - item - for item in documents - if item["kind"] == "Deployment" and item["metadata"]["name"] == name - ) - - -class _SwitchyardResponse: - """Minimal context-managed response used by routing contract tests.""" - - def __init__(self, selected: str, rationale: str = "local classifier vote"): - self.headers = { - "x-model-router-selected-model": selected, - "x-model-router-rationale": rationale, - } - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self): - return b"{}" - - def test_auto_lane_uses_switchyard_worker_decision(): observed = {} @@ -460,1512 +386,3 @@ def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): (5000, signal.SIGKILL), } assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) - assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] - - -def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): - task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") - calls = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( - ValueError("not a Git repository") - ), - block_task=lambda *_args, **kwargs: calls.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - - lanes.execute_claim("cassandra", "t_bad_worktree") - - assert calls[0]["kind"] == "capability" - assert calls[0]["expected_run_id"] == 7 - assert "not a Git repository" in calls[0]["reason"] - - -def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): - workspace = tmp_path / "workspace" - workspace.mkdir() - inside = workspace / "report.json" - outside = tmp_path / "auth.json" - inside.write_text("{}\n", encoding="utf-8") - outside.write_text("secret\n", encoding="utf-8") - - assert lanes.workspace_artifacts( - workspace, - ["report.json", str(outside), "missing.json"], - ) == [str(inside)] - - -def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_resume", - current_run_id=9, - assignee="cli-auto", - max_runtime_seconds=60, - ) - state_file = tmp_path / "state.json" - state_file.write_text( - json.dumps({"current_route": {"provider": "claude"}}), - encoding="utf-8", - ) - (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") - prompts = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "resume objective", - add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: None, - block_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "git_handoff", - lambda _workspace, output: f"HANDOFF:{output}", - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda _route, prompt, *_args, **_kwargs: ( - prompts.append(prompt) - or lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "blockers": [], - }, - False, - ) - ), - ) - - lanes.execute_claim("cassandra", "t_resume") - - assert "HANDOFF:prior provider evidence" in prompts[0] - - -def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): - route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) - command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") - assert "--dangerously-bypass-approvals-and-sandbox" in command - assert "--json" in command - assert "--output-schema" in command - assert 'model_reasoning_effort="xhigh"' in command - - claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} - claude = lanes._claude_command( - lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), - "Review.", - claude_state, - False, - ) - assert "--dangerously-skip-permissions" in claude - assert "--output-format" in claude and "stream-json" in claude - assert "--json-schema" in claude - assert "--disallowedTools" in claude - assert "Bash(kubectl apply *)" not in claude - assert "Bash(flux reconcile *)" not in claude - assert "max" not in claude - - -def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): - prompt = lanes.build_prompt("Review the change.", tmp_path) - - assert "put defects and risks in findings" in prompt - assert "blockers array must be empty whenever status is completed" in prompt - assert "findings" in lanes.RESULT_SCHEMA["properties"] - assert set(lanes.RESULT_SCHEMA["required"]) == set( - lanes.RESULT_SCHEMA["properties"] - ) - assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] - - -@pytest.mark.parametrize( - "command", - [ - "git push --force origin main", - "git reset --hard HEAD~1", - "git clean -fd", - ], -) -def test_claude_pretool_hook_blocks_hard_denies(command: str): - assert policy.denial_reason(command) - - -def test_claude_pretool_hook_allows_normal_engineering(): - assert policy.denial_reason("pytest -q testing/tests") is None - assert policy.denial_reason("git push origin feature/hermes") is None - assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None - assert policy.denial_reason("flux reconcile kustomization hermes") is None - assert policy.denial_reason("vault kv get kv/atlas/hermes") is None - - -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): - state = tmp_path / ".claude.json" - settings = tmp_path / "settings.json" - state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") - settings.write_text( - json.dumps( - { - "theme": "dark", - "permissions": { - "deny": [ - "Bash(kubectl apply *)", - "Bash(flux reconcile *)", - "Bash(vault kv *)", - "Bash(custom-owner-rule *)", - ] - }, - } - ) - + "\n", - encoding="utf-8", - ) - - client_config.configure_claude_state(state) - client_config.configure_claude_settings(settings) - - state_value = json.loads(state.read_text()) - settings_value = json.loads(settings.read_text()) - assert state_value["promptQueueUseCount"] == 4 - assert state_value["bypassPermissionsModeAccepted"] is True - assert settings_value["theme"] == "dark" - assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] - assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] - assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] - assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] - assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] - hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] - assert "claude_command_policy.py" in hook["command"] - - -def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): - session = tmp_path / "home/.config/herdr/session.json" - session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") - binary = tmp_path / "tools/bin/herdr" - binary.parent.mkdir(parents=True) - binary.write_text("legacy", encoding="utf-8") - (tmp_path / "home/.claude").mkdir() - (tmp_path / "home/.codex").mkdir() - - archive = migration.archive_legacy_state(tmp_path) - - value = json.loads(archive.read_text()) - assert value["legacy_session"]["agents"][0]["session_id"] == "abc" - assert (tmp_path / "home/.claude").is_dir() - assert (tmp_path / "home/.codex").is_dir() - assert not binary.exists() - assert not (tmp_path / "home/.config/herdr").exists() - - -def test_agent_uses_one_native_kanban_control_plane(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/balanced", - "model": "atlas/auto/balanced", - } - assert config["kanban"]["dispatch_in_gateway"] is True - assert config["kanban"]["default_assignee"] == "cli-auto" - assert config["plugins"]["enabled"] == ["auto-router"] - - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - assert pod["enableServiceLinks"] is False - names = {item["name"] for item in pod["containers"]} - assert "cli-lane-runner" in names - assert "terminal" in names - assert not any("herdr" in name for name in names) - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert "herdr server" not in rendered - assert "herdr-dispatch" not in rendered - - -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): - deployment = _agent_deployment() - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - lane = containers["cli-lane-runner"] - environment = {item["name"]: item["value"] for item in lane["env"]} - - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" - assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, - } - - -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ - pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( - item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] - if item["key"] == "kubernetes.io/hostname" - ) - - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) - assert hermes["resources"]["requests"]["cpu"] == "300m" - - -def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - containers = {item["name"]: item for item in pod["containers"]} - assert "webui" not in containers - - assert "dashboard" not in containers - hermes = containers["hermes"] - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" - assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" - assert hermes["command"] == ["/bin/sh", "-ec"] - startup = hermes["args"][0] - assert ". /opt/data/.env" in startup - assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_DASHBOARD"] == "1" - assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" - assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" - assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - assert hermes["securityContext"]["runAsUser"] == 0 - assert hermes["securityContext"]["runAsGroup"] == 0 - for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): - probe = hermes[probe_name] - assert probe["exec"]["command"] == [ - "curl", - "-fsS", - "http://127.0.0.1:9119/api/status", - ] - - terminal = containers["terminal"] - command = terminal["args"][0] - assert "--base-path /terminal" in command - assert "--check-origin" not in command - assert "/usr/bin/tmux new-session -A" in command - assert "--continue" in command - assert "--yolo" in command - terminal_env = {item["name"]: item["value"] for item in terminal["env"]} - assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - - claude_broker = containers["claude-broker"] - claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} - assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" - assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} - assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} - - args = containers["oauth2-proxy"]["args"] - terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" - dashboard_upstream = "--upstream=http://127.0.0.1:9119/" - assert terminal_upstream in args - assert dashboard_upstream in args - assert args.index(terminal_upstream) < args.index(dashboard_upstream) - assert "--pass-host-header=false" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" - ) - assert patch_init["command"][-1] == "/patched/server.py" - for name in ("hermes", "terminal"): - mounts = containers[name]["volumeMounts"] - assert { - "name": "tui-gateway-patch", - "mountPath": "/opt/hermes/tui_gateway/server.py", - "subPath": "server.py", - } in mounts - - ingress_documents = [ - item - for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) - if item - ] - middlewares = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Middleware" - } - assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ - "replacement" - ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" - ingresses = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Ingress" - } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" - - -def test_broker_services_survive_sibling_container_readiness_loss(): - services = _services() - - for name in ( - "hermes-image-broker", - "hermes-codex-broker", - "hermes-local-image", - "hermes-claude-broker", - ): - assert services[name]["spec"]["publishNotReadyAddresses"] is True - - -def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") - assert "eventsRetryAttempt.current" in dockerfile - assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile - assert "events feed rejected (${ev.code}) — reload the page" in dockerfile - assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile - assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile - assert dockerfile.count("' await api.getSessions(1, 0,") == 2 - assert ".then(() => gw.connect())" in dockerfile - assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile - assert "dashboard token rotated by a server restart" in dockerfile - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [{"protocol": "TCP", "port": 9010}], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) - - -def test_owner_agent_installs_the_pinned_operator_toolchain(): - script = (SCRIPTS / "install_agent_tools.sh").read_text() - for value in [ - "flux", - "helm", - "kustomize", - "jq", - "yq", - "gh", - "vault", - "sops", - "age", - "age-keygen", - "k9s", - "terraform", - "go", - "gofmt", - ]: - assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script - assert ( - "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" - in script - ) - assert script.count("sha256sum -c -") == 1 - - deployment = _agent_deployment() - installer = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "install-agent-tools" - ) - assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] - assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) - init_config = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - init_command = init_config["command"][2] - assert "# Hermes managed operator PATH." in init_command - assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command - assert 'chmod 0644 "${profile_file}"' in init_command - - -def test_owner_agent_uses_only_the_canonical_hostname(): - paths = [ - HERMES / "agent-configmap.yaml", - HERMES / "agent-deployment.yaml", - HERMES / "agent-ingress.yaml", - Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", - ] - for path in paths: - content = path.read_text() - assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content - - -def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - display = config["display"] - assert display["resume_exchanges"] >= 10000 - assert display["resume_max_user_chars"] >= 10000000 - assert display["resume_max_assistant_chars"] >= 10000000 - assert config["agent"]["max_turns"] == 180 - assert config["delegation"]["max_iterations"] == 120 - - -def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): - source = tmp_path / "auth.py" - destination = tmp_path / "patched/auth.py" - source.write_text( - 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', - encoding="utf-8", - ) - auth_patch.patch(source, destination) - content = destination.read_text() - assert 'os.environ.get("HERMES_AUTH_FILE"' in content - - -def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "auth.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - auth_patch.patch(source, tmp_path / "patched.py") - - -def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): - provider = tmp_path / "runtime_provider.py" - provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") - provider_out = tmp_path / "patched/runtime_provider.py" - codex_runtime_patch.patch_provider(provider, provider_out) - assert '"api_mode": "codex_app_server"' in provider_out.read_text() - - session = tmp_path / "codex_app_server_session.py" - session.write_text( - codex_runtime_patch.SESSION_SIGNATURE_BEFORE - + codex_runtime_patch.SESSION_REQUEST_BEFORE, - encoding="utf-8", - ) - session_out = tmp_path / "patched/codex_app_server_session.py" - codex_runtime_patch.patch_session(session, session_out) - session_content = session_out.read_text() - assert 'turn_params["model"] = model' in session_content - assert 'turn_params["effort"] = effort' in session_content - assert '"approvalPolicy": "never"' in session_content - assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content - - turn = tmp_path / "codex_runtime.py" - turn.write_text( - codex_runtime_patch.FALLBACK_CONTEXT_BEFORE - + codex_runtime_patch.TURN_BEFORE, - encoding="utf-8", - ) - turn_out = tmp_path / "patched/codex_runtime.py" - codex_runtime_patch.patch_turn(turn, turn_out) - turn_content = turn_out.read_text() - assert "model=str(getattr(agent" in turn_content - assert "build_cross_provider_codex_prompt" in turn_content - - fallback = tmp_path / "chat_completion_helpers.py" - fallback.write_text( - codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, - encoding="utf-8", - ) - fallback_out = tmp_path / "patched/chat_completion_helpers.py" - codex_runtime_patch.patch_fallback(fallback, fallback_out) - fallback_content = fallback_out.read_text() - assert 'agent.api_mode = "codex_app_server"' in fallback_content - assert "agent._codex_cross_provider_fallback = True" in fallback_content - - loop = tmp_path / "conversation_loop.py" - loop.write_text( - codex_runtime_patch.FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.STREAM_RECOVERY_BEFORE, - encoding="utf-8", - ) - loop_out = tmp_path / "patched/conversation_loop.py" - codex_runtime_patch.patch_loop(loop, loop_out) - loop_content = loop_out.read_text() - assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 - assert "build_cross_provider_codex_prompt" in loop_content - retry_dispatch = loop_content.index( - "Fallback activation happens inside this retry loop" - ) - api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) - assert api_kwargs == -1 or retry_dispatch < api_kwargs - assert "Provider stream ended before a complete response" in loop_content - assert "_is_transport_stub" in loop_content - assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content - - auxiliary = tmp_path / "auxiliary_client.py" - auxiliary.write_text( - codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - auxiliary_out = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) - auxiliary_content = auxiliary_out.read_text() - assert 'os.environ.get("CODEX_HOME"' in auxiliary_content - assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content - assert "never creates a metered API-key lane" in auxiliary_content - - -def test_agent_mounts_codex_auxiliary_runtime_patch(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-codex-runtime" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/auxiliary_client.py", - "/patched/auxiliary_client.py", - ] - expected_mount = { - "name": "codex-runtime-patch", - "mountPath": "/opt/hermes/agent/auxiliary_client.py", - "subPath": "auxiliary_client.py", - } - containers = {item["name"]: item for item in pod["containers"]} - for name in ("hermes", "terminal"): - assert expected_mount in containers[name]["volumeMounts"] - - -def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( - tmp_path: Path, - monkeypatch, -): - codex_home = tmp_path / ".codex" - codex_home.mkdir() - (codex_home / "auth.json").write_text( - json.dumps({"tokens": {"access_token": "cli-access-token"}}), - encoding="utf-8", - ) - monkeypatch.setenv("CODEX_HOME", str(codex_home)) - - source = tmp_path / "auxiliary_client.py" - source.write_text( - "import json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def read_token():\n" - " try:\n" - " raise RuntimeError('Hermes provider store intentionally empty')\n" - + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - destination = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(source, destination) - namespace: dict = {} - exec(compile(destination.read_text(), str(destination), "exec"), namespace) - - assert namespace["read_token"]() == "cli-access-token" - - -def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "runtime_provider.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") - - -def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): - config = tmp_path / "config.yaml" - config.write_text("model: {}\n", encoding="utf-8") - codex_home = tmp_path / ".codex" - calls = [] - - class Report: - errors = [] - - @staticmethod - def summary(): - return "configured" - - def migrate(value, **kwargs): - calls.append((value, kwargs)) - return Report() - - client_config.configure_codex_runtime(config, migrate, codex_home) - - assert calls[0][1]["default_permission_profile"] is None - assert calls[0][1]["codex_home"] == codex_home - content = (codex_home / "config.toml").read_text(encoding="utf-8") - assert 'approval_policy = "never"' in content - assert 'sandbox_mode = "danger-full-access"' in content - assert "default_permissions" not in content - - -def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): - config = tmp_path / "config.toml" - config.write_text( - 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', - encoding="utf-8", - ) - - client_config.configure_codex_owner_permissions(config) - client_config.configure_codex_owner_permissions(config) - - content = config.read_text(encoding="utf-8") - assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 - assert content.count('approval_policy = "never"') == 1 - assert content.count('sandbox_mode = "danger-full-access"') == 1 - assert "default_permissions" not in content - assert "[features]\nhooks = true" in content - - -def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): - source = tmp_path / "server.py" - destination = tmp_path / "patched/server.py" - source.write_text( - "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", - encoding="utf-8", - ) - - tui_gateway_patch.patch(source, destination) - - content = destination.read_text(encoding="utf-8") - assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content - assert 'configured = 180.0' in content - assert "return max(30.0, min(configured, 900.0))" in content - assert "timeout: float | None = None" in content - assert "ready.wait(timeout=wait_timeout)" in content - assert "def unchanged():" in content - - -def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "server.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - tui_gateway_patch.patch(source, tmp_path / "patched.py") - - -def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): - source = '' - content = ttyd_patch.patch_html(source) - assert 'id="atlas-ttyd-clipboard"' in content - assert "navigator.clipboard.writeText(text)" in content - assert "class AtlasRecoveringWebSocket" in content - assert "window.location.reload()" in content - assert "event.stopImmediatePropagation()" in content - - -def test_ttyd_patch_fails_closed_on_upstream_drift(): - with pytest.raises(RuntimeError, match="context changed"): - ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_cli_lanes_access.py b/testing/tests/test_hermes_cli_lanes_access.py new file mode 100644 index 00000000..01d9f4bf --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_access.py @@ -0,0 +1,378 @@ +"""Network, OAuth, Kubernetes, and SSH access contracts for Hermes CLI lanes.""" + +from __future__ import annotations + + +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + ROOT, + _agent_deployment, + _services, +) + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = {(item["kind"], item["name"]) for item in flux["spec"]["healthChecks"]} + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-agent-isolation" + ) + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "traefik"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-chat-tenant"}}}], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-switchyard"}}}], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [{"protocol": "TCP", "port": 9010}], + }, + ] + egress = isolation["spec"]["egress"] + assert any( + rule.get("ports") == [{"protocol": "TCP", "port": 9081}] for rule in egress + ) + assert "hermes-scm" in yaml.safe_dump(egress) + assert "gitea" in yaml.safe_dump(egress) + assert egress != [{}] + + +def test_owner_agent_has_scoped_read_only_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = ROOT / "services/hermes-observer-rbac/rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "hermes-agent-cluster-observer-v2", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-config"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"] + == "kv/data/atlas/hermes/developer-ssh" + ) + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())["resources"] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "sleep 300" in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = (HERMES / "scripts/node_account_hardening.py").read_text() + assert 'ACCOUNT = "hermes-agent"' in hardener + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in hardener + assert "ACCOUNT_UID = 1200" in hardener + assert "ACCOUNT_GID = 1200" in hardener + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next( + item for item in pod["containers"] if item["name"] == container_name + ) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text() + ) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert ( + state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"]["name"] + ) diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py new file mode 100644 index 00000000..bf1807f4 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -0,0 +1,444 @@ +"""Agent configuration and dashboard contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + _agent_deployment, + client_config, + lanes, + migration, + policy, +) + + +def test_workspace_preparation_failure_durably_blocks_the_claim( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") + calls = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( + ValueError("not a Git repository") + ), + block_task=lambda *_args, **kwargs: calls.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + + lanes.execute_claim("cassandra", "t_bad_worktree") + + assert calls[0]["kind"] == "capability" + assert calls[0]["expected_run_id"] == 7 + assert "not a Git repository" in calls[0]["reason"] + + +def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): + workspace = tmp_path / "workspace" + workspace.mkdir() + inside = workspace / "report.json" + outside = tmp_path / "auth.json" + inside.write_text("{}\n", encoding="utf-8") + outside.write_text("secret\n", encoding="utf-8") + + assert lanes.workspace_artifacts( + workspace, + ["report.json", str(outside), "missing.json"], + ) == [str(inside)] + + +def test_restart_provider_change_includes_explicit_workspace_handoff( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_resume", + current_run_id=9, + assignee="cli-auto", + max_runtime_seconds=60, + ) + state_file = tmp_path / "state.json" + state_file.write_text( + json.dumps({"current_route": {"provider": "claude"}}), + encoding="utf-8", + ) + (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") + prompts = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume objective", + add_comment=lambda *_args: None, + complete_task=lambda *_args, **_kwargs: None, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: f"HANDOFF:{output}", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda _route, prompt, *_args, **_kwargs: ( + prompts.append(prompt) + or lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "blockers": [], + }, + False, + ) + ), + ) + + lanes.execute_claim("cassandra", "t_resume") + + assert "HANDOFF:prior provider evidence" in prompts[0] + + +def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): + route = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () + ) + command = lanes._codex_command( + route, "Work.", tmp_path, {}, tmp_path / "result.json" + ) + assert "--dangerously-bypass-approvals-and-sandbox" in command + assert "--json" in command + assert "--output-schema" in command + assert 'model_reasoning_effort="xhigh"' in command + + claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} + claude = lanes._claude_command( + lanes.Route( + "claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, () + ), + "Review.", + claude_state, + False, + ) + assert "--dangerously-skip-permissions" in claude + assert "--output-format" in claude and "stream-json" in claude + assert "--json-schema" in claude + assert "--disallowedTools" in claude + assert "Bash(kubectl apply *)" not in claude + assert "Bash(flux reconcile *)" not in claude + assert "max" not in claude + + +def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): + prompt = lanes.build_prompt("Review the change.", tmp_path) + + assert "put defects and risks in findings" in prompt + assert "blockers array must be empty whenever status is completed" in prompt + assert "findings" in lanes.RESULT_SCHEMA["properties"] + assert set(lanes.RESULT_SCHEMA["required"]) == set( + lanes.RESULT_SCHEMA["properties"] + ) + assert ( + "assigned task itself" + in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] + ) + + +@pytest.mark.parametrize( + "command", + [ + "git push --force origin main", + "git reset --hard HEAD~1", + "git clean -fd", + ], +) +def test_claude_pretool_hook_blocks_hard_denies(command: str): + assert policy.denial_reason(command) + + +def test_claude_pretool_hook_allows_normal_engineering(): + assert policy.denial_reason("pytest -q testing/tests") is None + assert policy.denial_reason("git push origin feature/hermes") is None + assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None + assert policy.denial_reason("flux reconcile kustomization hermes") is None + assert policy.denial_reason("vault kv get kv/atlas/hermes") is None + + +def test_claude_settings_preserve_state_and_install_three_guardrail_layers( + tmp_path: Path, +): + state = tmp_path / ".claude.json" + settings = tmp_path / "settings.json" + state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") + settings.write_text( + json.dumps( + { + "theme": "dark", + "permissions": { + "deny": [ + "Bash(kubectl apply *)", + "Bash(flux reconcile *)", + "Bash(vault kv *)", + "Bash(custom-owner-rule *)", + ] + }, + } + ) + + "\n", + encoding="utf-8", + ) + + client_config.configure_claude_state(state) + client_config.configure_claude_settings(settings) + + state_value = json.loads(state.read_text()) + settings_value = json.loads(settings.read_text()) + assert state_value["promptQueueUseCount"] == 4 + assert state_value["bypassPermissionsModeAccepted"] is True + assert settings_value["theme"] == "dark" + assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] + assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] + assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] + assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] + assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] + hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] + assert "claude_command_policy.py" in hook["command"] + + +def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): + session = tmp_path / "home/.config/herdr/session.json" + session.parent.mkdir(parents=True) + session.write_text( + '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8" + ) + binary = tmp_path / "tools/bin/herdr" + binary.parent.mkdir(parents=True) + binary.write_text("legacy", encoding="utf-8") + (tmp_path / "home/.claude").mkdir() + (tmp_path / "home/.codex").mkdir() + + archive = migration.archive_legacy_state(tmp_path) + + value = json.loads(archive.read_text()) + assert value["legacy_session"]["agents"][0]["session_id"] == "abc" + assert (tmp_path / "home/.claude").is_dir() + assert (tmp_path / "home/.codex").is_dir() + assert not binary.exists() + assert not (tmp_path / "home/.config/herdr").exists() + + +def test_agent_uses_one_native_kanban_control_plane(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/balanced", + "model": "atlas/auto/balanced", + } + assert config["kanban"]["dispatch_in_gateway"] is True + assert config["kanban"]["default_assignee"] == "cli-auto" + assert config["plugins"]["enabled"] == ["auto-router"] + + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + assert pod["enableServiceLinks"] is False + names = {item["name"] for item in pod["containers"]} + assert "cli-lane-runner" in names + assert "terminal" in names + assert not any("herdr" in name for name in names) + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert "herdr server" not in rendered + assert "herdr-dispatch" not in rendered + + +def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): + deployment = _agent_deployment() + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + lane = containers["cli-lane-runner"] + environment = {item["name"]: item["value"] for item in lane["env"]} + + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert lane["resources"] == { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "2", "memory": "6Gi"}, + } + + +def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): + """Placement correction: keep the agent off nodes that cannot hold it. + + titan-04 is cordoned after repeated kernel undervoltage and kubelet + failure, and titan-19 was probe/Longhorn unstable under worker load, so + both must join the existing hard exclusions. That leaves titan-05 as the + healthy candidate, which is tight enough on requested CPU that the main + container has to give back 50m to schedule there. + """ + pod = _agent_deployment()["spec"]["template"]["spec"] + hostnames = next( + item + for item in pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + if item["key"] == "kubernetes.io/hostname" + ) + + assert hostnames["operator"] == "NotIn" + assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + assert hermes["resources"]["requests"]["cpu"] == "300m" + + +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + assert "dashboard" not in containers + hermes = containers["hermes"] + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" + assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" + assert hermes["command"] == ["/bin/sh", "-ec"] + startup = hermes["args"][0] + assert ". /opt/data/.env" in startup + assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_DASHBOARD"] == "1" + assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" + assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" + assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + assert hermes["securityContext"]["runAsUser"] == 0 + assert hermes["securityContext"]["runAsGroup"] == 0 + for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): + probe = hermes[probe_name] + assert probe["exec"]["command"] == [ + "curl", + "-fsS", + "http://127.0.0.1:9119/api/status", + ] + + terminal = containers["terminal"] + command = terminal["args"][0] + assert "--base-path /terminal" in command + assert "--check-origin" not in command + assert "/usr/bin/tmux new-session -A" in command + assert "--continue" in command + assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("hermes", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts + + ingress_documents = [ + item + for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if item + ] + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert ( + middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] + == "http://127.0.0.1:9119" + ) + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ( + ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + ) + assert ( + ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-terminal-slash@kubernetescrd" + ) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py new file mode 100644 index 00000000..279b74c5 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_kanban.py @@ -0,0 +1,392 @@ +"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_lanes_support import ( + lanes, +) + + +def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): + task = SimpleNamespace(id="t_auto", assignee=None, status="ready") + assigned = [] + + class Connection: + def close(self): + return None + + def assign_task(_conn, task_id, profile): + assigned.append((task_id, profile)) + task.assignee = profile + return True + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + assign_task=assign_task, + get_task=lambda _conn, _task_id: task, + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] + assert assigned == [("t_auto", "cli-auto")] + + +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( + monkeypatch, capsys +): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err + + +def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + def recompute_ready(connection): + if connection.board == "cassandra": + raise lanes.sqlite3.OperationalError("disk I/O error") + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(board), + recompute_ready=recompute_ready, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + error = capsys.readouterr().err + assert "temporarily skipping Kanban board 'cassandra'" in error + assert "storage OperationalError: disk I/O error" in error + + +def test_board_call_retries_storage_faults_on_fresh_connections(): + connections = [] + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + attempts = [] + + def operation(_connection): + attempts.append(1) + if len(attempts) < 3: + raise lanes.sqlite3.OperationalError("disk I/O error") + return "healthy" + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + ) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" + assert len(connections) == 3 + assert all(connection.closed for connection in connections) + + +@pytest.mark.parametrize( + ("result", "expected_action"), + [ + (lanes.ProcessResult(0, "plain text only", None, False), "block"), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "blockers": [], + }, + False, + ), + "complete", + ), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "The full test suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "blockers": [], + }, + False, + ), + "block", + ), + ], +) +def test_claim_requires_structured_evidence_and_surfaces_artifacts( + tmp_path: Path, + monkeypatch, + result, + expected_action, +): + task = SimpleNamespace( + id="t_worker", + current_run_id=4, + assignee="cli-auto", + max_runtime_seconds=60, + ) + calls = [] + heartbeats = [] + connections = [] + artifact = tmp_path / "reports/result.json" + artifact.parent.mkdir() + artifact.write_text("{}\n", encoding="utf-8") + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "bounded objective", + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + + def run_provider(*args, **_kwargs): + assert connections[0].closed + before_heartbeat = len(connections) + assert args[6]("working") is True + assert len(connections) == before_heartbeat + 1 + assert connections[-1].closed + return result + + monkeypatch.setattr(lanes, "run_provider", run_provider) + + lanes.execute_claim("cassandra", "t_worker") + + assert calls[0][0] == expected_action + assert heartbeats == [("working", 4)] + if expected_action == "complete": + assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] + assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] + else: + assert calls[0][1]["kind"] == "capability" + assert all(connection.closed for connection in connections) + + +def test_goal_card_continues_after_local_judge_rejects_progress( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_goal", + current_run_id=12, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + calls = [] + comments = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "state_path", + lambda _board, _task_id: tmp_path / "state.json", + ) + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () + ) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "Focused tests passed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "Full tests passed; commit pushed and remote HEAD verified.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter( + [ + (False, "commit, push, and remote verification are missing"), + (True, "all explicit acceptance criteria have evidence"), + ] + ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + judge_goal_completion, + ) + + lanes.execute_claim("cassandra", "t_goal") + + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert any( + "Goal completion rejected; continuing turn 2/3" in item for item in comments + ) + assert any( + "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments + ) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] + assert reports == [] diff --git a/testing/tests/test_hermes_cli_lanes_support.py b/testing/tests/test_hermes_cli_lanes_support.py new file mode 100644 index 00000000..242351b9 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_support.py @@ -0,0 +1,79 @@ +"""Shared loaders and manifest helpers for Hermes CLI-lane contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +HERMES = ROOT / "services/hermes" +KEYCLOAK = ROOT / "services/keycloak" +FLUX_HERMES = ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +policy = _load("claude_command_policy") +migration = _load("migrate_herdr_state") +auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") +codex_runtime_patch = _load("patch_codex_runtime") +ttyd_patch = _load("patch_ttyd_index") +client_config = _load("configure_agent_clients") + + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py new file mode 100644 index 00000000..2ed232cb --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -0,0 +1,386 @@ +"""Operator toolchain and runtime-patch contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + tui_gateway_patch, + ttyd_patch, +) + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert "configured = 180.0" in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index bc2d5b5f..683c0201 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -2,44 +2,20 @@ from __future__ import annotations -import importlib.util import json import subprocess -import sys import tomllib from pathlib import Path -import pytest import yaml - -SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -sys.path.insert(0, str(SCRIPT.parent)) -routing = importlib.import_module("hermes_model_routing") -catalog_resolver = importlib.import_module("routing_catalog") -SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) -assert SPEC and SPEC.loader -coordinator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = coordinator -SPEC.loader.exec_module(coordinator) - - -def _base_config() -> dict: - """Return the minimal coordinator configuration used by routing tests.""" - return { - "model": { - "provider": "openai-codex", - "default": "gpt-5.6-terra", - "model": "gpt-5.6-terra", - }, - "fallback_providers": [ - {"provider": "anthropic", "model": "claude-opus-5"}, - dict(routing.LOCAL_FALLBACK), - ], - "toolsets": ["kanban"], - "terminal": {"cwd": "/opt/data/workspace"}, - } - +from testing.tests.test_hermes_coordinator_support import ( + SCRIPT, + _base_config, + catalog_resolver, + coordinator, + routing, +) def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) @@ -334,177 +310,3 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") - - -def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( - tmp_path: Path, monkeypatch -): - """A damaged project board degrades Kanban instead of stopping all services.""" - - class FakeKanbanDbCorruptError(Exception): - def __init__(self): - super().__init__("integrity check failed") - self.db_path = tmp_path / "kanban.db" - self.backup_path = tmp_path / "kanban.db.corrupt.backup" - self.reason = "row out of order" - - def raise_corruption(_root: Path) -> None: - raise FakeKanbanDbCorruptError() - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - status = coordinator.bootstrap_cassandra_state(tmp_path) - - assert status == { - "state": "corrupt-preserved", - "database": str(tmp_path / "kanban.db"), - "backup": str(tmp_path / "kanban.db.corrupt.backup"), - "reason": "row out of order", - } - - -def test_unrelated_cassandra_bootstrap_failure_remains_fatal( - tmp_path: Path, monkeypatch -): - """Only the known fail-closed corruption state may be degraded.""" - - class FakeKanbanDbCorruptError(Exception): - pass - - def raise_permission_error(_root: Path) -> None: - raise PermissionError("cannot access board") - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - try: - coordinator.bootstrap_cassandra_state(tmp_path) - except PermissionError as error: - assert str(error) == "cannot access board" - else: - raise AssertionError("unrelated board errors must fail the coordinator refresh") - - -def test_cassandra_workspace_uses_valid_configured_worktree( - tmp_path: Path, monkeypatch -): - """The coordinator selects an explicit Git worktree but rejects stale paths.""" - active = tmp_path / "cassandra-v69" - active.mkdir() - (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) - - assert coordinator.cassandra_workspace() == active - - (active / ".git").unlink() - assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH - - -@pytest.mark.parametrize( - ("current_remote", "repair_action"), - [ - ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), - (None, "add"), - (coordinator.CASSANDRA_REMOTE, None), - ], - ids=["old-origin", "missing-origin", "canonical-origin"], -) -def test_cassandra_sync_repairs_existing_worktree( - tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None -): - """Existing worktrees use the canonical Atlas origin before fetching.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - token_path = tmp_path / "gitea-token" - token_path.write_text("configured\n", encoding="utf-8") - monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) - commands: list[list[str]] = [] - environments: list[dict[str, str]] = [] - - def run(command, **kwargs): - commands.append(command) - environments.append(kwargs.get("env", {})) - if command[-2:] == ["get-url", "origin"]: - return subprocess.CompletedProcess( - command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" - ) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" - repair_commands = [command for command in commands if repair_action in command] - assert bool(repair_commands) is bool(repair_action) - assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] - assert all("GITEA_TOKEN" not in environment for environment in environments) - - -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - commands: list[list[str]] = [] - - def run(command, **_kwargs): - commands.append(command) - stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None - return subprocess.CompletedProcess(command, 0, stdout=stdout) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - state = coordinator.sync_cassandra_repo({}) - - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] - - -def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): - """A project switch moves queued work without relocating active evidence.""" - - class Task: - def __init__(self, task_id: str, status: str): - self.id = task_id - self.status = status - self.workspace_kind = "dir" - self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) - - class Closing: - def __enter__(self): - return object() - - def __exit__(self, *_args): - return None - - class FakeKanban: - tasks = [ - Task("queued", "todo"), - Task("active", "running"), - Task("done", "done"), - ] - moved: list[tuple[str, str]] = [] - - @staticmethod - def connect_closing(*, board: str): - assert board == "cassandra" - return Closing() - - @classmethod - def list_tasks(cls, _connection, *, include_archived: bool): - assert include_archived is False - return cls.tasks - - @classmethod - def set_workspace_path(cls, _connection, task_id: str, path: Path): - cls.moved.append((task_id, str(path))) - - active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") - coordinator._migrate_open_cassandra_tasks(FakeKanban, active) - - assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_cassandra.py b/testing/tests/test_hermes_coordinator_cassandra.py new file mode 100644 index 00000000..646db291 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_cassandra.py @@ -0,0 +1,194 @@ +"""Cassandra workspace and board contracts for the Hermes coordinator.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator + + +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + + +def test_cassandra_workspace_uses_valid_configured_worktree( + tmp_path: Path, monkeypatch +): + """The coordinator selects an explicit Git worktree but rejects stale paths.""" + active = tmp_path / "cassandra-v69" + active.mkdir() + (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) + + assert coordinator.cassandra_workspace() == active + + (active / ".git").unlink() + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + token_path = tmp_path / "gitea-token" + token_path.write_text("configured\n", encoding="utf-8") + monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + + def run(command, **kwargs): + commands.append(command) + environments.append(kwargs.get("env", {})) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + assert all("GITEA_TOKEN" not in environment for environment in environments) + + +def test_cassandra_sync_repairs_origin_through_broker_without_token( + tmp_path: Path, monkeypatch +): + """Repository sync uses the credential-isolated broker remote.""" + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "explicitly-unavailable-token") + ) + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready" + assert any( + "hermes-scm-broker.hermes-scm.svc.cluster.local" in " ".join(cmd) + for cmd in commands + ) + assert all("token" not in " ".join(cmd).lower() for cmd in commands) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + + +def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): + """A project switch moves queued work without relocating active evidence.""" + + class Task: + def __init__(self, task_id: str, status: str): + self.id = task_id + self.status = status + self.workspace_kind = "dir" + self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) + + class Closing: + def __enter__(self): + return object() + + def __exit__(self, *_args): + return None + + class FakeKanban: + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] + moved: list[tuple[str, str]] = [] + + @staticmethod + def connect_closing(*, board: str): + assert board == "cassandra" + return Closing() + + @classmethod + def list_tasks(cls, _connection, *, include_archived: bool): + assert include_archived is False + return cls.tasks + + @classmethod + def set_workspace_path(cls, _connection, task_id: str, path: Path): + cls.moved.append((task_id, str(path))) + + active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") + coordinator._migrate_open_cassandra_tasks(FakeKanban, active) + + assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_coverage.py b/testing/tests/test_hermes_coordinator_coverage.py new file mode 100644 index 00000000..fe4f3168 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_coverage.py @@ -0,0 +1,267 @@ +"""Behavioral branch coverage for Hermes project coordination.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +from contextlib import nullcontext +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator, routing + + +def test_workspace_falls_back_when_unset_or_invalid(tmp_path: Path, monkeypatch): + monkeypatch.delenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", raising=False) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(tmp_path / "missing")) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +def test_task_migration_is_noop_for_base_workspace(): + class Board: + @staticmethod + def connect_closing(**_kwargs): + raise AssertionError("base workspace must not access Kanban") + + coordinator._migrate_open_cassandra_tasks(Board, coordinator.CASSANDRA_BASE_PATH) + + +class _Project: + id = "project-id" + + +def _fake_hermes_modules(monkeypatch, *, existing: bool, active: bool): + events = [] + kb = types.ModuleType("hermes_cli.kanban_db") + kb.board_exists = lambda slug: existing + kb.create_board = lambda *args, **kwargs: events.append( + ("create-board", args, kwargs) + ) + kb.set_current_board = lambda slug: events.append(("current", slug)) + kb.connect_closing = lambda **kwargs: nullcontext(object()) + kb.list_tasks = lambda *_a, **_k: [] + + pdb = types.ModuleType("hermes_cli.projects_db") + pdb.connect_closing = lambda: nullcontext(object()) + pdb.get_project = lambda *_a: _Project() if existing else None + pdb.get_active_id = lambda *_a: "active" if active else None + pdb.create_project = ( + lambda *args, **kwargs: events.append(("create-project", kwargs)) or "new-id" + ) + pdb.set_active = lambda *args: events.append(("active", args[-1])) + pdb.update_project = lambda *args, **kwargs: events.append(("update", kwargs)) + pdb.add_folder = lambda *args, **kwargs: events.append(("folder", args[-1], kwargs)) + + package = types.ModuleType("hermes_cli") + package.kanban_db = kb + package.projects_db = pdb + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kb) + monkeypatch.setitem(sys.modules, "hermes_cli.projects_db", pdb) + return events + + +@pytest.mark.parametrize( + ("existing", "active", "expected"), + [ + (False, False, {"current", "create-project", "active"}), + (False, True, {"current", "create-project"}), + (True, True, {"update", "folder"}), + ], +) +def test_bootstrap_cassandra_creates_or_updates_state( + tmp_path: Path, monkeypatch, existing, active, expected +): + events = _fake_hermes_modules(monkeypatch, existing=existing, active=active) + base = tmp_path / "projects/cassandra" + worktree = tmp_path / "projects/cassandra-worktree" + worktree.mkdir(parents=True) + (worktree / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", base) + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(worktree)) + coordinator.bootstrap_cassandra(tmp_path / "hermes") + names = {event[0] for event in events} + assert expected <= names + assert "create-board" in names + + +def test_bootstrap_state_ready_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda _root: None) + assert coordinator.bootstrap_cassandra_state(tmp_path) == {"state": "ready"} + + +def test_corruption_type_imports_pinned_exception(monkeypatch): + class Expected(Exception): + pass + + package = types.ModuleType("hermes_cli") + kanban = types.ModuleType("hermes_cli.kanban_db") + kanban.KanbanDbCorruptError = Expected + package.kanban_db = kanban + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kanban) + assert coordinator._kanban_corruption_type() is Expected + + +def test_repo_sync_reports_missing_git_and_unmanaged_directory( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: None) + assert coordinator.sync_cassandra_repo({}) == "git-unavailable" + + workspace = tmp_path / "cassandra" + workspace.mkdir() + (workspace / "user-file").write_text("preserve", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + assert coordinator.sync_cassandra_repo({}) == "unmanaged-nonempty-directory" + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + ("get-url-error", "remote-repair-failed"), + ("repair-exit", "remote-repair-failed-9"), + ("repair-error", "remote-repair-failed"), + ("fetch-exit", "sync-failed-8"), + ("fetch-error", "sync-failed"), + ], +) +def test_existing_repo_sync_reports_each_failure( + tmp_path: Path, monkeypatch, failure, expected +): + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + + def run(command, **_kwargs): + action = command[-2:] if len(command) >= 2 else [] + if action == ["get-url", "origin"]: + if failure == "get-url-error": + raise OSError("git unavailable") + return subprocess.CompletedProcess(command, 0, stdout="old\n") + if "set-url" in command: + if failure == "repair-error": + raise subprocess.TimeoutExpired(command, 30) + return subprocess.CompletedProcess( + command, 9 if failure == "repair-exit" else 0 + ) + if failure == "fetch-error": + raise OSError("fetch unavailable") + return subprocess.CompletedProcess(command, 8 if failure == "fetch-exit" else 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({}) == expected + + +@pytest.mark.parametrize( + ("exit_code", "expected"), [(0, "ready"), (7, "sync-failed-7")] +) +def test_empty_repo_clones_through_broker( + tmp_path: Path, monkeypatch, exit_code, expected +): + workspace = tmp_path / "cassandra" + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + seen = [] + + def run(command, **kwargs): + seen.append((command, kwargs["env"])) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert ( + coordinator.sync_cassandra_repo( + {"API_SERVER_KEY": "not-forwarded", "SAFE": "yes"} + ) + == expected + ) + assert coordinator.CASSANDRA_REMOTE in seen[0][0] + assert seen[0][1]["SAFE"] == "yes" + assert "API_SERVER_KEY" not in seen[0][1] + + +def _catalog(provider): + return routing.Catalog(provider, ["model"], True, True, "connected") + + +def test_refresh_once_writes_complete_status(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "_read_env", lambda _path: {"SAFE": "yes"}) + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: _catalog("codex")) + monkeypatch.setattr( + coordinator, "discover_claude_models", lambda: _catalog("claude") + ) + monkeypatch.setattr( + coordinator, "configure_routes", lambda *_a: {"coordinator": ["auto"]} + ) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda _root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda _env: "ready") + monkeypatch.setattr( + coordinator, "cassandra_workspace", lambda: tmp_path / "worktree" + ) + written = [] + monkeypatch.setattr( + coordinator, "_atomic_write", lambda path, value: written.append((path, value)) + ) + status = coordinator.refresh_once(tmp_path) + assert status["projects"]["cassandra"]["state"] == "ready" + assert json.loads(written[0][1])["providers"]["codex"]["connected"] is True + + +def test_main_once_success_and_failure(monkeypatch, capsys): + status = {"providers": {"codex": {"state": "ready"}}} + monkeypatch.setattr(sys, "argv", ["coordinator", "--once"]) + monkeypatch.setattr(coordinator, "refresh_once", lambda _root: status) + assert coordinator.main() == 0 + assert "codex=ready" in capsys.readouterr().out + monkeypatch.setattr( + coordinator, + "refresh_once", + lambda _root: (_ for _ in ()).throw(RuntimeError("failed")), + ) + assert coordinator.main() == 1 + assert "RuntimeError" in capsys.readouterr().out + + +@pytest.mark.parametrize(("interval", "expected"), [(1, 300), (400, 400)]) +def test_main_loop_sleeps_with_floor(monkeypatch, interval, expected): + calls = iter(({"providers": {}}, KeyboardInterrupt())) + + def refresh(_root): + value = next(calls) + if isinstance(value, BaseException): + raise value + return value + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr( + sys, "argv", ["coordinator", "--loop", "--interval", str(interval)] + ) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [expected] + + +def test_main_loop_recovers_from_refresh_failure(monkeypatch): + calls = iter((RuntimeError("temporary"), KeyboardInterrupt())) + + def refresh(_root): + error = next(calls) + raise error + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr(sys, "argv", ["coordinator", "--loop", "--interval", "300"]) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [300] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py new file mode 100644 index 00000000..2a41f070 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_support.py @@ -0,0 +1,35 @@ +"""Shared loader and configuration for Hermes coordinator contracts.""" + +from __future__ import annotations + +import importlib +import importlib.util +import sys +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" +sys.path.insert(0, str(SCRIPT.parent)) +routing = importlib.import_module("hermes_model_routing") +catalog_resolver = importlib.import_module("routing_catalog") +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) +assert SPEC and SPEC.loader +coordinator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = coordinator +SPEC.loader.exec_module(coordinator) + + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py new file mode 100644 index 00000000..362e8726 --- /dev/null +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -0,0 +1,183 @@ +"""Behavioral branch coverage for the Gitea protection policy checker.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + + +def _module(name: str = "branch_protection_coverage"): + return _load_path( + name, + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + + +def _rule(module, name: str = "main", **updates): + value = { + "rule_name": name, + "priority": 1, + "created_at": "2026-01-01T00:00:00Z", + **module._required("bstein"), + } + value.update(updates) + return value + + +def test_read_bounded_accepts_regular_file_and_closes_descriptor( + tmp_path: Path, monkeypatch +): + module = _module("branch_read_regular") + source = tmp_path / "rules.json" + source.write_bytes(b"[]") + closed: list[int] = [] + real_close = os.close + + def close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(module.os, "close", close) + assert module._read_bounded(source) == b"[]" + assert len(closed) == 1 + + +@pytest.mark.parametrize("kind", ["directory", "oversized", "short"]) +def test_read_bounded_rejects_unsafe_or_changed_input( + tmp_path: Path, monkeypatch, kind +): + module = _module(f"branch_read_{kind}") + source = tmp_path / "rules" + source.write_bytes(b"[]") + real_fstat = module.os.fstat + real_read = module.os.read + + if kind == "directory": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFDIR | 0o700, 0, 0, 0, 0, 0, 2, 0, 0, 0) + ), + ) + elif kind == "oversized": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFREG | 0o600, 0, 0, 0, 0, 0, module.MAX_INPUT + 1, 0, 0, 0) + ), + ) + else: + monkeypatch.setattr(module.os, "fstat", real_fstat) + monkeypatch.setattr(module.os, "read", lambda descriptor, maximum: b"[") + with pytest.raises(module.PolicyError): + module._read_bounded(source) + monkeypatch.setattr(module.os, "read", real_read) + + +@pytest.mark.parametrize( + ("pattern", "branch", "expected"), + [ + ("m?in", "main", True), + ("m[ai]in", "main", True), + ("m[!z]in", "main", True), + ("m]ain", "m]ain", True), + ("m}ain", "m}ain", True), + ("m[a-z]in", "main", True), + ("m{ain,aster}", "main", True), + ("m\\ain", "main", True), + ], +) +def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): + module = _module("branch_glob_supported") + assert module._matches(pattern, branch) is expected + + +@pytest.mark.parametrize( + "pattern", + [ + "m\\", + "m[", + "m[]", + "m[!]", + "m[[a]", + "m[\\a]", + "m[-a]", + "m[a-]", + "m[a-b-c]", + "m[z-a]", + "m[aa-b]", + "m{ain}", + "m{ain,}", + "m{ain,aster", + ], +) +def test_invalid_globs_are_literal_nonmatches(pattern): + module = _module("branch_glob_invalid") + assert module._matches(pattern, "main") is False + + +@pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) +def test_rule_names_are_ascii_and_bounded(pattern): + module = _module("branch_name_bounds") + with pytest.raises(module.PolicyError, match="rule name"): + module._matches(pattern, "main") + + +@pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) +def test_created_at_requires_short_timezone_aware_iso(value): + module = _module("branch_created_bounds") + with pytest.raises(module.PolicyError, match="creation time"): + module._created(value) + + +@pytest.mark.parametrize( + "value", + [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], +) +def test_evaluate_rejects_invalid_top_level_inputs(value): + module = _module("branch_evaluate_top") + with pytest.raises((module.PolicyError, json.JSONDecodeError)): + module.evaluate(value, "main", "bstein") + + +def test_evaluate_rejects_unknown_branch_and_excess_rules(): + module = _module("branch_evaluate_bounds") + with pytest.raises(module.PolicyError, match="input"): + module.evaluate(b"[]", "release", "bstein") + with pytest.raises(module.PolicyError, match="response"): + module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") + + +@pytest.mark.parametrize("priority", [True, 0, 1_000_001, "1"]) +def test_evaluate_rejects_ambiguous_priority_types(priority): + module = _module("branch_priority_bounds") + rule = _rule(module, priority=priority) + with pytest.raises(module.PolicyError, match="priority"): + module.evaluate(json.dumps([rule]).encode(), "main", "bstein") + + +def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): + module = _module("branch_main_paths") + source = tmp_path / "rules.json" + source.write_text(json.dumps([_rule(module)]), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["checker", str(source), "main", "bstein"]) + assert module.main() == 0 + assert capsys.readouterr().out.strip() == "PRESENT" + + source.write_text("not-json", encoding="utf-8") + assert module.main() == 1 + assert "branch protection check failed" in capsys.readouterr().err + + monkeypatch.setattr( + module, "_read_bounded", lambda _path: (_ for _ in ()).throw(OSError()) + ) + assert module.main() == 1 diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py new file mode 100644 index 00000000..f811827c --- /dev/null +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -0,0 +1,375 @@ +"""Behavioral coverage for internal safe-Gitea client and policy paths.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +import urllib.error +import urllib.parse +from email.message import Message +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_gitea_support import CLIENT_PATH, Response, _load +from testing.tests.test_hermes_scm_broker_support import _load_path + + +def _policy(name: str = "gitea_policy_coverage"): + return _load_path(name, CLIENT_PATH.parent / "gitea_api_policy.py") + + +def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): + module = _load() + token = tmp_path / "token" + token.write_text(" synthetic-runtime-value \n", encoding="utf-8") + assert module.read_token(token) == "synthetic-runtime-value" + token.write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + module.read_token(token) + + monkeypatch.setenv("GITEA_BASE_URL", module.CANONICAL_BASE_URL) + assert module.configured_base_url() == module.CANONICAL_BASE_URL + monkeypatch.setenv("GITEA_BASE_URL", "https://example.invalid") + with pytest.raises(module.PolicyError, match="fixed"): + module.configured_base_url() + + +def test_safe_urlopen_delegates_only_to_redirect_rejecting_opener(monkeypatch): + module = _load() + sentinel = object() + monkeypatch.setattr( + module._SAFE_OPENER, + "open", + lambda request, timeout: (request, timeout, sentinel), + ) + request = object() + assert module._safe_urlopen(request, 7) == (request, 7, sentinel) + + +def test_api_target_and_request_body_edge_paths(monkeypatch): + module = _load() + with pytest.raises(module.PolicyError, match="path exceeds"): + module._split_api_path("/api/v1/" + "x" * 510) + with pytest.raises(module.PolicyError, match="request body"): + module.authorize_request( + "GET", "/api/v1/repos/atlas/cassandra", {"unexpected": True} + ) + with pytest.raises(module.PolicyError, match="query"): + module.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls?page=1", + {}, + ) + with pytest.raises(module.PolicyError, match="only read"): + module.authorize_request("TRACE", "/api/v1/repos/atlas/cassandra/pulls", None) + with pytest.raises(module.PolicyError, match="accepts only"): + module.authorize_request("POST", "/api/v1/repos/atlas/cassandra/pulls", []) + + +def test_response_status_fallback_and_nested_fail_closed(): + module = _load() + + class GetCodeResponse(Response): + def __init__(self): + super().__init__({"name": "cassandra"}, status=200) + del self.status + + def getcode(self): + return 200 + + assert ( + module.read( + "/api/v1/repos/atlas/cassandra", + token="synthetic", + opener=lambda *_a, **_k: GetCodeResponse(), + ) + == b'{"name": "cassandra"}' + ) + with pytest.raises(module.PolicyError, match="omitted required"): + module._nested({"base": None}, "base", "ref") + + +@pytest.mark.parametrize("body", [b"not-json", b"[]"]) +def test_create_response_requires_json_object(body: bytes): + module = _load() + with pytest.raises(module.PolicyError, match="invalid pull-request metadata"): + module._require_create_response( + body, + repo="cassandra", + base="main", + head="feature/test", + head_sha="a" * 40, + title="WIP: Test", + body="Evidence", + ) + + +def test_write_body_handles_empty_newline_and_missing_newline(monkeypatch): + module = _load() + + class Buffer: + value = bytearray() + + @classmethod + def write(cls, value): + cls.value.extend(value) + + monkeypatch.setattr(module.sys, "stdout", types.SimpleNamespace(buffer=Buffer)) + module._write_body(b"") + module._write_body(b"one\n") + module._write_body(b"two") + assert bytes(Buffer.value) == b"one\ntwo\n" + + +def test_main_executes_broker_read_and_create_paths(monkeypatch): + module = _load() + outputs: list[bytes] = [] + client = types.ModuleType("scm_broker_client") + client.read = lambda path: json.dumps({"path": path}).encode() + client.create_draft = lambda repo, **data: json.dumps( + {"repo": repo, **data}, sort_keys=True + ).encode() + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + monkeypatch.setattr(module, "_write_body", outputs.append) + + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 0 + assert json.loads(outputs.pop()) == {"path": "/api/v1/repos/atlas/cassandra"} + assert ( + module.main( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "feature/coverage", + "--head-sha", + "a" * 40, + "--title", + "Coverage repair", + "--body", + "Review the focused tests", + ] + ) + == 0 + ) + assert json.loads(outputs.pop())["repo"] == "cassandra" + + +def test_main_handles_http_and_policy_failures(monkeypatch, capsys): + module = _load() + client = types.ModuleType("scm_broker_client") + + def http_failure(_path): + headers = Message() + raise urllib.error.HTTPError("url", 503, "unavailable", headers, None) + + client.read = http_failure + client.create_draft = lambda *_a, **_k: b"{}" + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + assert "HTTP 503" in capsys.readouterr().err + client.read = lambda _path: (_ for _ in ()).throw(module.PolicyError("rejected")) + assert module.main(["read", "/not-allowed"]) == 1 + assert "no credential" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("function", "value"), + [ + ("_validate_repo", "."), + ("_validate_repo", ".."), + ("_validate_sha", "short"), + ("_validate_pr_number", True), + ("_validate_pr_number", 0), + ("_validate_pr_number", 2_147_483_648), + ], +) +def test_policy_scalar_validators_reject_ambiguous_values(function, value): + module = _policy("gitea_policy_scalar") + with pytest.raises(module.PolicyError): + getattr(module, function)(value) + + +def test_ref_bounds_cover_utf8_controls_git_failure_and_success(monkeypatch): + module = _policy("gitea_policy_refs") + with pytest.raises(module.PolicyError, match="valid UTF-8"): + module._validate_ref_bounds("\ud800", "head") + with pytest.raises(module.PolicyError, match="UTF-8"): + module._validate_ref_bounds("🧪" * 64, "head") + with pytest.raises(module.PolicyError, match="safe same"): + module._validate_ref("bad\nref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 1), + ) + with pytest.raises(module.PolicyError, match="same-repository"): + module._validate_ref("invalid-ref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 0), + ) + assert module._validate_ref("feature/valid", "head") == "feature/valid" + + +@pytest.mark.parametrize( + ("value", "required", "match"), + [ + (None, False, "must be text"), + (" ", True, "must not be empty"), + ("\ud800", False, "valid UTF-8"), + ("x\x00y", False, "safe request limit"), + ], +) +def test_text_validation_rejects_nontext_empty_invalid_and_controls( + value, required, match +): + module = _policy("gitea_policy_text") + with pytest.raises(module.PolicyError, match=match): + module._validate_text(value, "field", 20, 20, required=required) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("password", True), + ("accountKey", True), + ("registry-key", True), + ("clientEmail", True), + ("clientId", True), + ("accessId", True), + ("connectionString", True), + ("dockerConfigJson", True), + ("release_note", False), + ], +) +def test_sensitive_key_semantics_cover_generic_forms(key, expected): + module = _policy("gitea_policy_keys") + assert module._is_sensitive_key(key) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", False), + ('"synthetic"', True), + ("{encoded}", True), + ("Bearer value-12345678", True), + ("ghp_" + "a" * 24, True), + ("https://example.invalid/value", True), + ("name@example.invalid", True), + ("${RUNTIME_VALUE}", True), + ("AbCdEf0123456789", True), + ("reject empty values", False), + ("ordinary", True), + ], +) +def test_assignment_value_shape_distinguishes_prose_from_credentials(value, expected): + module = _policy("gitea_policy_assignment_values") + assert module._looks_sensitive_assignment_value(value) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("single", False), + ("add regression coverage", True), + ("add regression_coverage", False), + ("ordinary engineering prose", False), + ], +) +def test_prose_classifier_covers_length_punctuation_and_lead_words(value, expected): + module = _policy("gitea_policy_prose") + assert module._looks_like_prose(value) is expected + + +def test_structured_json_walk_covers_lists_nonstring_keys_and_limits(): + module = _policy("gitea_policy_json_walk") + assert module._json_value_has_sensitive_assignment({1: "ignored"}) is False + assert module._json_value_has_sensitive_assignment([{"release": "safe"}]) is False + assert ( + module._json_value_has_sensitive_assignment({"client_secret": "value"}) is True + ) + assert ( + module._json_value_has_sensitive_assignment( + {"outer": {"client_secret": "value"}} + ) + is True + ) + assert ( + module._json_value_has_sensitive_assignment({"type": "service-account"}) is True + ) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], depth=33) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], nodes=[2048]) + + +def test_json_key_decoder_and_embedded_document_scanner_cover_failures(): + module = _policy("gitea_policy_json_decoder") + assert module._decode_json_key("client\\u005fsecret") == ("client_secret", True) + decoded, valid = module._decode_json_key("client\\qsecret") + assert valid is False and "client" in decoded + assert list(module._decoded_json_documents("prose only")) == [] + assert list(module._decoded_json_documents("x {bad y [1, 2]")) == [[1, 2]] + with pytest.raises(module.PolicyError, match="scan limit"): + list(module._decoded_json_documents("{" * 33)) + assert ( + module._has_structured_sensitive_assignment('"client\\qsecret": value') is True + ) + assert ( + module._has_structured_sensitive_assignment("type:\n service-account") is True + ) + + +def test_high_entropy_detector_covers_long_mixed_and_low_entropy_values(): + module = _policy("gitea_policy_entropy") + assert module._has_high_entropy_token("A1_" * 100) is True + assert ( + module._has_high_entropy_token( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/_=" + ) + is True + ) + assert module._has_high_entropy_token("a" * 64) is False + assert module._has_high_entropy_token("short prose") is False + + +@pytest.mark.parametrize( + "query", + [ + "bad", + "a=1&b=2&c=3&d=4", + "page=1&page=2", + "=value", + "unexpected=1", + "page=abc", + "page=01", + "page=10001", + "limit=51", + "state=merged", + ], +) +def test_query_validator_covers_each_rejection_class(query): + module = _policy("gitea_policy_query") + target = urllib.parse.urlsplit("/api/v1/repos/atlas/cassandra/pulls?" + query) + with pytest.raises(module.PolicyError): + module._validate_query(target, {"page", "limit", "state"}) + + +def test_draft_title_rejects_prefix_without_content(): + module = _policy("gitea_policy_empty_draft") + with pytest.raises(module.PolicyError, match="after the draft prefix"): + module._draft_title("WIP:") + + +def test_query_validator_rejects_noncanonical_raw_form(): + module = _policy("gitea_policy_raw_query") + target = urllib.parse.SplitResult("", "", "/api/v1/repos/atlas/cassandra", "%", "") + with pytest.raises(module.PolicyError, match="canonical ASCII"): + module._validate_query(target, set()) diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 447f3067..ae419e10 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -1,75 +1,12 @@ -"""Adversarial contracts for the Atlas-only draft pull-request client.""" +"""Route and request-boundary contracts for the Atlas Gitea client.""" from __future__ import annotations -import copy -import importlib.util -import json -import sys import urllib.request -from pathlib import Path import pytest -ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" -HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" - - -def _load(): - spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def _draft_payload(**updates): - payload = { - "base": "main", - "body": "Review evidence", - "head": "hermes/review-fix", - "title": "WIP: Repair review findings", - } - payload.update(updates) - return payload - - -def _draft_response(**updates): - response = { - "number": 3, - "state": "open", - "draft": True, - "merged": False, - "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "title": "WIP: Focused fix", - "body": "Review evidence", - "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, - "head": { - "ref": "hermes/fix", - "sha": HEAD_SHA, - "repo": {"full_name": "atlas/cassandra"}, - }, - } - response.update(updates) - return response - - -class Response: - def __init__(self, body: object): - self.body = body if isinstance(body, bytes) else json.dumps(body).encode() - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self, limit=-1): - return self.body if limit < 0 else self.body[:limit] - +from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): client = _load() @@ -101,6 +38,10 @@ def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): ("https://evil.example", "/api/v1/repos/atlas/cassandra"), ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev/", "/api/v1/repos/atlas/cassandra"), + ("HTTPS://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://SCM.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://user@scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), @@ -114,6 +55,72 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1\nHost: evil.example", + "/api/v1/repos/atlas/cassandra/pulls/1\r\nX-Test: value", + "/api/v1/repos/atlas/cassandra/pulls/1\tignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x00ignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x1fignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x7fignored", + "/api/v1/repos/atlas/cassandra\\pulls\\1", + "/api/v1/repos/atlas/cassandra/pulls/%31", + "/api/v1/repos/atlas/cassandra/pulls/１", + " https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + ], +) +def test_raw_noncanonical_target_is_rejected_before_urlsplit_and_opener( + path: str, monkeypatch +): + client = _load() + split_called = False + opener_called = False + original_urlsplit = client.urllib.parse.urlsplit + + def urlsplit(*args, **kwargs): + nonlocal split_called + split_called = True + return original_urlsplit(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + monkeypatch.setattr(client.urllib.parse, "urlsplit", urlsplit) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert split_called is False + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1?", + "//scm.bstein.dev/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/./pulls/1", + "/api/v1/repos/atlas/cassandra/../admin", + "/api/v1/repos/atlas/cassandra//pulls/1", + "/api/v1/repos/atlas/cassandra/pulls/1?limit=01", + ], +) +def test_noncanonical_round_trip_or_segments_never_reach_opener(path: str): + client = _load() + opener_called = False + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert opener_called is False + + @pytest.mark.parametrize( "path", [ @@ -180,226 +187,103 @@ def test_read_query_is_bounded(path: str): @pytest.mark.parametrize( - ("method", "path", "data"), + "query", [ - ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), - ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), - ( - "POST", - "/api/v1/repos/atlas/cassandra/pulls/4/reviews", - {"event": "APPROVED"}, - ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), - ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + "page=0", + "page=10001", + "page=01", + "limit=0", + "limit=51", + "limit=01", + "page=" + "9" * 4000, + "page=０", + "page=%EF%BC%90", + "state=%6fpen", + "p%61ge=1", + "page=1&page=2", + "page=1&" + "x" * 17 + "=1", + "state=" + "x" * 17, + "page=1&limit=2&state=open&extra=3", ], ) -def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( - method: str, path: str, data: object -): +def test_read_query_requires_canonical_bounded_ascii(query: str): client = _load() with pytest.raises(client.PolicyError): - client.build_request( - method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + client.authorize_request( + "GET", f"/api/v1/repos/atlas/cassandra/pulls?{query}", None ) @pytest.mark.parametrize( - "ref", + "suffix", [ - "foo/.bar", - "foo/bar.lock/baz", - "foo..bar", - "foo@{bar", - "foo//bar", - "-danger", - "danger.", - "danger~one", - "danger^one", - "danger:one", - "danger one", + "pulls/{number}", + "pulls/{number}.patch", + "pulls/{number}.diff", + "pulls/{number}/commits", + "pulls/{number}/files", + "commits/{number}/status", + "commits/{number}/statuses", + "statuses/{number}", + "branches/{number}", ], ) -def test_complete_git_ref_validation_rejects_invalid_names(ref: str): - client = _load() - - with pytest.raises(client.PolicyError): - client._validate_ref(ref, "head") - - -def test_git_ref_validation_uses_fixed_trusted_binary(): - client = _load() - - assert client.GIT_BIN == "/usr/bin/git" - assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" - - -def test_create_forces_draft_title_and_same_repository_branch_names(): - client = _load() - assert ( - client.authorize_request( - "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() - ) - == "create-draft" - ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(title="Not a draft"), - ) - with pytest.raises(client.PolicyError): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(head="someone:branch"), - ) - - -def test_runtime_token_is_only_an_authorization_header(): - client = _load() - request = client.build_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - base_url=client.CANONICAL_BASE_URL, - token="do-not-leak", - data=_draft_payload(), - ) - - assert "do-not-leak" not in request.full_url - assert b"do-not-leak" not in request.data - assert request.get_header("Authorization") == "token do-not-leak" - - -@pytest.mark.parametrize( - "body", - [ - "pass" + "word=not-a-real-credential", - "Authorization: " + "Bearer not-a-real-credential-value", - "token: " + "ghp_" + "notarealcredentialvalue123456", - "-----BEGIN OPENSSH " + "PRIVATE KEY-----", - "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", - ], -) -def test_body_rejects_common_credential_shapes(body: str): - client = _load() - - with pytest.raises(client.PolicyError, match="credential material"): - client._validate_body(body) - - -def test_create_rejects_exact_runtime_token_before_network(): +def test_oversized_numeric_or_captured_path_never_reaches_opener(suffix: str): client = _load() called = False def opener(*_args, **_kwargs): nonlocal called called = True - return Response(_draft_response()) + return Response(b"{}") - with pytest.raises(client.PolicyError, match="runtime credential"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="accidental runtime-sentinel value", - token="runtime-sentinel", + path = "/api/v1/repos/atlas/cassandra/" + suffix.format(number="9" * 4000) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert called is False + + +@pytest.mark.parametrize( + "number", + ["0", "01", "2147483648", "１２", "%31", "12345678901"], +) +@pytest.mark.parametrize("tail", ["", ".patch", ".diff", "/commits", "/files"]) +def test_noncanonical_or_out_of_range_pr_number_never_reaches_opener( + number: str, tail: str +): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read( + f"/api/v1/repos/atlas/cassandra/pulls/{number}{tail}", + token="runtime", opener=opener, ) assert called is False -def test_create_verifies_every_server_postcondition(): +def test_maximum_bounded_pr_number_is_readable(): client = _load() - calls = [] + called = False - def opener(request, timeout): - calls.append((request, timeout)) - return Response(_draft_response()) + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") - result = client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="Review evidence", - token="runtime", - opener=opener, - ) - - assert json.loads(result) == _draft_response() - payload = json.loads(calls[0][0].data) - assert payload == { - "base": "main", - "body": "Review evidence", - "head": "hermes/fix", - "title": "WIP: Focused fix", - } - assert calls[0][1] == 30 - - -def test_create_postcondition_rejects_every_material_mismatch(): - client = _load() - mutations = [ - ("number", 0), - ("state", "closed"), - ("draft", False), - ("merged", True), - ("html_url", "https://evil.example/pulls/3"), - ("url", "https://evil.example/api/pulls/3"), - ("title", "Focused fix"), - ("body", "different"), - ] - documents = [] - for key, value in mutations: - document = _draft_response() - document[key] = value - documents.append(document) - for path, value in [ - (("base", "ref"), "master"), - (("base", "repo", "full_name"), "evil/cassandra"), - (("head", "ref"), "other"), - (("head", "sha"), "0" * 40), - (("head", "repo", "full_name"), "evil/cassandra"), - ]: - document = copy.deepcopy(_draft_response()) - target = document - for key in path[:-1]: - target = target[key] - target[path[-1]] = value - documents.append(document) - - for document in documents: - with pytest.raises(client.PolicyError): - client._require_create_response( - json.dumps(document).encode(), - repo="cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="WIP: Focused fix", - body="Review evidence", - ) - - -def test_read_response_is_bounded(): - client = _load() - - with pytest.raises(client.PolicyError, match="safe size limit"): + assert ( client.read( - "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls/2147483647", token="runtime", - opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + opener=opener, ) - - -def test_output_redaction_covers_exact_token_and_authorization_header(): - client = _load() - raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' - redacted = client.redact_bytes(raw, "do-not-leak") - - assert b"do-not-leak" not in redacted - assert redacted.count(b"[REDACTED]") >= 1 + == b"{}" + ) + assert called is True diff --git a/testing/tests/test_hermes_gitea_pr_creation.py b/testing/tests/test_hermes_gitea_pr_creation.py new file mode 100644 index 00000000..ad2967ed --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_creation.py @@ -0,0 +1,203 @@ +"""Draft-creation input contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_payload, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client._validate_ref.__globals__["GIT_BIN"] == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + +@pytest.mark.parametrize("field", ["base", "head"]) +@pytest.mark.parametrize("oversized", ["r" * 100_000, "🧪" * 128]) +def test_oversized_ref_never_invokes_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not receive an oversized ref") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + refs = {"base": "main", "head": "hermes/fix"} + refs[field] = oversized + with pytest.raises(client.PolicyError, match="branch-name limit"): + client.create_draft( + "cassandra", + base=refs["base"], + head=refs["head"], + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + ("field", "oversized"), + [("repo", "r" * 101), ("title", "🧪" * 200), ("body", "🧪" * 9_000)], +) +def test_other_text_bounds_fail_before_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run before cheap input bounds") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + values = { + "repo": "cassandra", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = oversized + with pytest.raises(client.PolicyError): + client.create_draft( + values["repo"], + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" diff --git a/testing/tests/test_hermes_gitea_pr_dlp.py b/testing/tests/test_hermes_gitea_pr_dlp.py new file mode 100644 index 00000000..4a2b1d27 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_dlp.py @@ -0,0 +1,317 @@ +"""Credential-loss-prevention contracts for Atlas draft creation.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + "sensitive", + [ + "client_" + "secret=not-a-real-value", + '{"client_' + 'secret":\n"synthetic-value"}', + '{"client_' + 'secret"\n:\n"synthetic-value"}', + '{"client\\u005f' + 'secret":"synthetic-value"}', + "client_" + "secret:\n synthetic-value", + '{"client\\q' + 'secret":"synthetic-value"}', + '{"client\n' + 'secret":"synthetic-value"}', + '"client_' + 'secret"\x0b:\n"synthetic-value"', + "ACCESS_" + "TOKEN = 'not-a-real-value'", + '{"refresh_' + 'token": "not-a-real-value"}', + "private_" + "key: not-a-real-value", + "AWS_SECRET_ACCESS_" + "KEY=not-a-real-value", + "aws_access_key_" + "id: not-a-real-value", + "AWS_SESSION_" + "TOKEN = 'not-a-real-value'", + '{"AccessKey' + 'Id":"not-a-real-value"}', + '{"SecretAccess' + 'Key":"not-a-real-value"}', + '{"Session' + 'Token":"not-a-real-value"}', + "aws-security-" + "token: not-a-real-value", + "Account" + "Key=not-a-real-value", + "SharedAccess" + "Signature: not-a-real-value", + "AZURE_STORAGE_CONNECTION_" + "STRING='not-a-real-value'", + "DefaultEndpointsProtocol=https;AccountName=fake;Account" + + "Key=not-a-real-value;EndpointSuffix=example", + "?sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2099-01-01&sig=" + "not-a-real-value", + "DOCKER_AUTH_" + "CONFIG='not-a-real-value'", + '{"auths":{"registry.example":{"auth":"bm90LXJlYWw="}}}', + '{"identity' + 'token":"not-a-real-value"}', + '{"type":"service_' + 'account","client_email":"fake@example.test"}', + '{"private_key_' + 'id":"not-a-real-value"}', + '{"client_' + 'email":"fake@example.test"}', + "GOOGLE_CREDENTIALS" + "=not-a-real-value", + "personal_access_" + "token: not-a-real-value", + "GITEA_" + "TOKEN=not-a-real-value", + "gitlab-token" + ": not-a-real-value", + "pat" + "=not-a-real-value", + "Authorization: " + "Bearer not-a-real-credential-value", + "authorization = " + '"Basic not-a-real-credential-value"', + "Bearer" + "=not-a-real-credential-value", + "Basic" + ": not-a-real-credential-value", + "pass" + "word=not-a-real-credential", + "ghp_" + "notarealcredentialvalue123456", + "github_pat_" + "notarealcredentialvalue123456", + "glpat-" + "notarealcredentialvalue123456", + "xoxb-" + "not-a-real-credential-value-123456", + "sk-ant-" + "notarealcredentialvalue123456", + "sk-proj-" + "notarealcredentialvalue123456", + "sk_live_" + "notarealcredentialvalue123456", + "ya29." + "notarealcredentialvalue123456", + "gta_" + "notarealcredentialvalue123456", + "whsec_" + "notarealcredentialvalue123456", + "npm_" + "notarealcredentialvalue123456", + "pypi-" + "notarealcredentialvalue123456789012345", + "hf_" + "notarealcredentialvalue123456", + "SG." + "notarealvalue1234" + ".notarealcredentialvalue123456", + "SK" + "a" * 32, + "https://hooks.slack.com/services/" + "T000/B000/notarealvalue123456", + "https://discord.com/api/webhooks/123456789/" + "notarealcredentialvalue123456", + "https://fake.webhook.office.com/" + "notarealcredentialvalue123456", + "webhook_" + "url=https://example.test/not-real", + "FutureCloudSigning" + "Credential=not-a-real-value", + "future-client-signing-" + "key: not-a-real-value", + "future_client_signing_" + "key='not-a-real-value'", + '{"serviceAccountPrivate' + 'Key":"not-a-real-value"}', + "CONTAINER_REGISTRY_" + "CREDENTIAL=not-a-real-value", + "someWebhookSigning" + "Secret: not-a-real-value", + "client" + "Key=not-a-real-value", + "session" + "Key: not-a-real-value", + "access" + "Id=not-a-real-value", + "credentials" + ": {user: fake}", + "private" + "Key: |", + "nuget_api_" + "key=not-a-real-value", + "oy2" + "a" * 44, + "sk_test_" + "notarealcredentialvalue123456", + "A1b2C3d4E5f6G7h8I9j0K_l-M+n/O=pQ2rS3tU4vW5xY6zZ7aB8cC9d", + "AKIA" + "A" * 16, + "AIza" + "a" * 35, + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "ssh-ed25519 " + "bm90YXJlYWxjcmVkZW50aWFsdmFsdWU=", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_expanded_credential_shapes_before_network( + sensitive: str, field: str +): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert called is False + + +def test_very_long_compact_body_is_rejected_before_request_or_network(): + client = _load() + request_built = False + opener_called = False + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + client.build_request = build_request + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="a" * 300, + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "safe_text", + [ + "AWS_SECRET_ACCESS_KEY is injected at runtime", + "Token: reject empty values", + "Authorization = preserve header behavior", + "Password: add regression", + "Review the Authorization header behavior", + "Bearer authentication is required for this route", + "Document Docker auths payload rejection", + "The client_email field belongs to service accounts", + "AccountKey assignments must be blocked", + "This patch changes token validation without including a value", + "FutureCloudSigningCredential handling needs a regression test", + "The clientKey name is documented without an assigned value", + "A SHA-256 digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef is evidence", + ], +) +def test_credential_policy_keeps_normal_engineering_prose_usable(safe_text: str): + client = _load() + + assert client._validate_body(safe_text) == safe_text + assert client._draft_title(safe_text) == f"WIP: {safe_text}" + + +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_exact_runtime_token_before_network(field: str): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + runtime_token = "exact-random-runtime-sentinel-7b73ac61" + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = f"Accidental {runtime_token} value" + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert request_built is False + assert called is False + + +@pytest.mark.parametrize("field", ["repo", "base", "head", "title", "body"]) +def test_every_public_field_rejects_exact_runtime_token_before_git_or_network( + field: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + runtime_token = "runtime-sentinel" + values = { + "repo": "cassandra", + "base": "main", + "head": "hermes/fix", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = runtime_token + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run for a credential-bearing field") + + def build_request(*_args, **_kwargs): + nonlocal request_built + request_built = True + raise AssertionError("a request must not be built") + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + raise AssertionError("the opener must not be called") + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + monkeypatch.setattr(client, "build_request", build_request) + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + values["repo"], + base=values["base"], + head=values["head"], + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "sensitive", + [ + " ".join(["{}"] * 32) + ' {"client_secret":"synthetic-value"}', + "prefix_" + "ghp_" + "notarealcredentialvalue123456_suffix", + "client_secret: correct horse battery staple", + "'client_secret':\n synthetic-value", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_structured_scanner_closes_bounded_and_wrapped_token_bypasses( + sensitive: str, field: str +): + client = _load() + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert called is False diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index d3b47eef..ee482cfe 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -13,8 +13,10 @@ import pytest import yaml ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) def _load(): @@ -96,12 +98,12 @@ def test_http_error_path_redacts_token(monkeypatch, capsys): assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 captured = capsys.readouterr() assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err + assert "credential was disclosed" in captured.err -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): +def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source + assert "scm_broker_client" in client_source assert "GITEA_TOKEN" not in client_source deployment = yaml.safe_load( @@ -109,9 +111,7 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): ) template = deployment["spec"]["template"] annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) + assert not any("gitea" in key.lower() for key in annotations) runtime = next( volume for volume in template["spec"]["volumes"] @@ -136,6 +136,17 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): kustomization = yaml.safe_load( (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") ) + common = yaml.safe_load( + (ROOT / "services/hermes/scm-common/kustomization.yaml").read_text( + encoding="utf-8" + ) + ) + boundary = common["configMapGenerator"][0] + assert boundary["name"] == "hermes-scm-boundary" + assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] + assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] + assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] + assert not any("gitea_askpass" in item for item in boundary["files"]) generator = next( item for item in kustomization["configMapGenerator"] @@ -145,3 +156,19 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", ] + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + pod = broker["spec"]["template"] + assert pod["spec"]["serviceAccountName"] == "hermes-scm-broker" + assert pod["metadata"]["annotations"][ + "vault.hashicorp.com/agent-inject-secret-gitea-token" + ] == "kv/data/atlas/hermes/developer-gitea" + assert not any( + volume.get("hostPath") or volume.get("persistentVolumeClaim") + for volume in pod["spec"]["volumes"] + ) diff --git a/testing/tests/test_hermes_gitea_pr_postconditions.py b/testing/tests/test_hermes_gitea_pr_postconditions.py new file mode 100644 index 00000000..c06a7784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_postconditions.py @@ -0,0 +1,199 @@ +"""Response and postcondition contracts for Atlas draft creation.""" + +from __future__ import annotations + +import base64 +import copy +import json + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + +@pytest.mark.parametrize("status", [200, 202, 204, 206]) +def test_create_accepts_only_http_201(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response(), status=status), + ) + + +@pytest.mark.parametrize("status", [201, 202, 204, 206]) +def test_read_accepts_only_http_200(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"{}", status=status), + ) + + +def test_successful_create_validates_each_ref_once(monkeypatch): + client = _load() + calls = [] + + def git_run(command, **_kwargs): + calls.append(command) + return type("Result", (), {"returncode": 0})() + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response()), + ) + + assert [command[-1] for command in calls] == [ + "refs/heads/main", + "refs/heads/hermes/fix", + ] + + +def test_create_verifies_every_server_postcondition(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + return Response(_draft_response()) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 + + +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("number", 2_147_483_648), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): + client = _load() + + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + ) + + +def test_direct_api_rejects_unexpected_success_content_type(): + client = _load() + response = Response(b"{}") + response.headers.replace_header("Content-Type", "text/html") + + with pytest.raises(client.PolicyError, match="unexpected response type"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: response, + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +@pytest.mark.parametrize( + "reflected", + [ + b"runtime-sentinel", + base64.b64encode(b"runtime-sentinel"), + base64.b64encode(b"hermes-automation:runtime-sentinel"), + ], +) +def test_direct_api_rejects_credential_reflection(reflected: bytes): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b'{"unexpected":"' + reflected + b'"}' + ), + ) diff --git a/testing/tests/test_hermes_gitea_support.py b/testing/tests/test_hermes_gitea_support.py new file mode 100644 index 00000000..8b00e784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_support.py @@ -0,0 +1,75 @@ +"""Shared fixtures for the Atlas-only Gitea client contracts.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + +class Response: + def __init__(self, body: object, status: int | None = None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = ( + status if status is not None else (201 if isinstance(body, dict) else 200) + ) + self.headers = Message() + self.headers["Content-Type"] = "application/json" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] diff --git a/testing/tests/test_hermes_node_account_hardening.py b/testing/tests/test_hermes_node_account_hardening.py new file mode 100644 index 00000000..9cf2bb3a --- /dev/null +++ b/testing/tests/test_hermes_node_account_hardening.py @@ -0,0 +1,252 @@ +"""Core contracts for the dedicated Hermes node SSH identity.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + SCRIPT, + _fixture, + _key, + _load, +) + +def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key( + tmp_path: Path, monkeypatch +): + module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + + module.reconcile(public_key) + first = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + module.reconcile(public_key) + second = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + + assert first == second + expected = module._expected_records() + for name, original in originals.items(): + assert first[name] == original + ":".join(expected[name]) + "\n" + backup = module.HOST_ETC / f"{name}.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == original + assert expected["shadow"][1] == "!" + assert expected["group"][-1] == "" + assert "disk:x:6:atlas\n" in first["group"] + assert "sudo:x:27:oceanus\n" in first["group"] + + hermes_keys = ( + module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert hermes_keys == key + "\n" + for user in module.LEGACY_ACCOUNTS: + legacy = ( + module.HOST_HOME / user / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert legacy == f"{other} {user}\n" + backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n" + + +def test_identity_conflict_fails_closed_without_editing_account_databases( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + conflict = originals["passwd"] + ( + f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n" + ) + (module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="identity conflicts"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict + for name in ("group", "shadow", "gshadow"): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name] + assert not (module.HOST_HOME / module.ACCOUNT).exists() + + +def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="invalid record"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[ + "passwd" + ] + assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n" + + +def test_preexisting_home_fails_before_account_database_or_key_changes( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + home = module.HOST_HOME / module.ACCOUNT + home.mkdir() + (home / "unrelated").write_text("preserve\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="home already exists"): + module.reconcile(public_key) + + for name, original in originals.items(): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original + assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists() + assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n" + + +def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8") + ) + ) + daemonset = next(item for item in documents if item["kind"] == "DaemonSet") + spec = daemonset["spec"]["template"]["spec"] + assert spec["tolerations"] == [{"operator": "Exists"}] + command = spec["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in command + assert "sleep 300" in command + mounts = {item["name"]: item for item in spec["volumes"]} + assert mounts["host-home"]["hostPath"]["path"] == "/home" + assert mounts["host-etc"]["hostPath"]["path"] == "/etc" + assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s" + assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet" + assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s" + assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd" + assert "usermod" not in command + assert "groupmod" not in command + + policies = list( + yaml.safe_load_all( + (ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8") + ) + ) + isolation = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-node-ssh-access-isolation" + ) + assert isolation["spec"]["ingress"] == [] + assert isolation["spec"]["egress"] == [] + + source = SCRIPT.read_text(encoding="utf-8") + assert "ACCOUNT = \"hermes-agent\"" in source + assert "ACCOUNT_UID = 1200" in source + assert "ACCOUNT_GID = 1200" in source + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source + assert "audit_membership" in source + image = spec["containers"][0]["image"] + assert image == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1" + + +def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes( + tmp_path: Path, monkeypatch +): + module = _load() + sensitive = tmp_path / "k3s" + sensitive.mkdir(mode=0o755) + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + # Production requires uid 0. The test process owns its fixture but runs the + # real xattr/ACL implementation using the test owner's uid as that boundary. + + module._deny_sensitive_root(sensitive) + first = module._read_acl(sensitive) + module._deny_sensitive_root(sensitive) + second = module._read_acl(sensitive) + + assert first == second + entries = module._decode_acl(first, 0o755) + assert (module.ACL_USER, 0, 1200) in entries + assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries + backup = host_etc / "hermes-node-boundary/k3s.acl" + assert backup.read_bytes() == b"N" + assert backup.stat().st_mode & 0o777 == 0o600 + + +def test_sensitive_root_rejects_non_directory_and_non_root_owner( + tmp_path: Path, monkeypatch +): + module = _load() + regular = tmp_path / "not-a-directory" + regular.write_text("preserve", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unsafe sensitive directory"): + module._deny_sensitive_root(regular) + + directory = tmp_path / "not-root-owned" + directory.mkdir() + if directory.stat().st_uid == 0: + pytest.skip("cannot construct a non-root-owned fixture as root") + with pytest.raises(module.HardeningError, match="not root-owned"): + module._deny_sensitive_root(directory) + + +def test_acl_parser_rejects_malformed_or_incomplete_values(): + module = _load() + with pytest.raises(module.HardeningError, match="malformed"): + module._decode_acl(b"too short", 0o755) + incomplete = module._encode_acl( + [(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)] + ) + with pytest.raises(module.HardeningError, match="incomplete"): + module._decode_acl(incomplete, 0o755) + + +def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + public_key = tmp_path / "public-key" + public_key.write_text( + _key(b"synthetic-key") + "\n", + encoding="utf-8", + ) + moved = False + monkeypatch.setattr(module, "_reconcile_databases", lambda: None) + + def fail_acl(): + raise module.HardeningError("ACL unavailable") + + def move_key(_path): + nonlocal moved + moved = True + + monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl) + monkeypatch.setattr(module, "_move_key", move_key) + + with pytest.raises(module.HardeningError, match="ACL unavailable"): + module.reconcile(public_key) + assert moved is False + + +def test_runtime_ssh_config_forces_dedicated_account_for_every_titan(): + stage = ( + ROOT / "services/hermes/scripts/stage_runtime_access.py" + ).read_text(encoding="utf-8") + assert '"Host titan-*\\n User hermes-agent\\n"' in stage + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + assert "User atlas" not in yaml.safe_dump(agent) + assert "User oceanus" not in yaml.safe_dump(agent) diff --git a/testing/tests/test_hermes_node_account_io.py b/testing/tests/test_hermes_node_account_io.py new file mode 100644 index 00000000..6b05cc0a --- /dev/null +++ b/testing/tests/test_hermes_node_account_io.py @@ -0,0 +1,222 @@ +"""Crash-safety and metadata contracts for Hermes node-account writes.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + _fixture, + _load, + _typed_key, +) + +def test_key_identity_ignores_options_and_comments_and_removes_legacy_first( + tmp_path: Path, monkeypatch +): + module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + key_fields = key.split() + decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment' + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module.reconcile(public_key) + + key_writes = [path for path in writes if path.name == "authorized_keys"] + assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys") + assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1]) + + +def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human") + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{modern}\n{key}\n", encoding="utf-8") + + module.reconcile(public_key) + + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + assert path.read_text() == modern + "\n" + + +def test_crash_during_legacy_key_removal_never_installs_duplicate_key( + tmp_path: Path, monkeypatch +): + module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_write = module.atomic_write + key_writes = 0 + + def crash_on_second_key(path, value, metadata): + nonlocal key_writes + if path.name == "authorized_keys": + key_writes += 1 + if key_writes == 2: + raise module.HardeningError("synthetic crash") + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", crash_on_second_key) + with pytest.raises(module.HardeningError, match="synthetic crash"): + module.reconcile(public_key) + assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists() + + +def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled( + tmp_path: Path, monkeypatch +): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + " existing\n") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + if path.name == "authorized_keys": + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module._move_key(public_key) + + assert writes[0] == target + assert writes[-1] == target + assert all(path != target for path in writes[1:-1]) + assert target.read_text() == key + "\n" + + +def test_account_database_write_preserves_mode_owner_and_xattrs( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + passwd = module.HOST_ETC / "passwd" + passwd.chmod(0o640) + os.setxattr(passwd, "user.hermes-test", b"preserve") + before = passwd.stat() + + module.reconcile(public_key) + + after = passwd.stat() + assert after.st_mode & 0o777 == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + assert os.getxattr(passwd, "user.hermes-test") == b"preserve" + backup = module.HOST_ETC / "passwd.hermes-boundary-backup" + assert os.getxattr(backup, "user.hermes-test") == b"preserve" + assert backup.read_text() == originals["passwd"] + + +def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch): + module = _load() + io_module = sys.modules[module.atomic_write.__module__] + target = tmp_path / "passwd" + target.write_bytes(b"old\n") + metadata = io_module.FileSnapshot( + value=b"old\n", + device=target.stat().st_dev, + inode=target.stat().st_ino, + mode=0o600, + uid=os.getuid(), + gid=os.getgid(), + size=4, + mtime_ns=target.stat().st_mtime_ns, + ctime_ns=target.stat().st_ctime_ns, + xattrs=( + ("system.posix_acl_access", b"synthetic-acl"), + ("security.selinux", b"synthetic-label"), + ("user.audit", b"synthetic-xattr"), + ), + ) + restored: list[tuple[str, bytes]] = [] + monkeypatch.setattr( + io_module.os, + "setxattr", + lambda _fd, name, value: restored.append((name, value)), + ) + + io_module.atomic_write(target, b"new\n", metadata) + + assert target.read_bytes() == b"new\n" + assert restored == list(metadata.xattrs) + + +def test_concurrent_database_change_is_detected_before_any_account_write( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_backup = module.backup_once + backup_count = 0 + writes: list[Path] = [] + + def racing_backup(path, snapshot, maximum): + nonlocal backup_count + result = real_backup(path, snapshot, maximum) + backup_count += 1 + if backup_count == 4: + group = module.HOST_ETC / "group" + group.write_text(originals["group"] + "race:x:4000:\n") + return result + + monkeypatch.setattr(module, "backup_once", racing_backup) + monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path)) + with pytest.raises(module.HardeningError, match="concurrent host account change"): + module.reconcile(public_key) + assert writes == [] + + +def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority(): + observer = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text() + ) + hermes = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text() + ) + bindings = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text() + ) + assert "dependsOn" not in observer["spec"] + assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= { + "hermes-observer-rbac", + "hermes-scm-broker", + } + assert {item["name"] for item in bindings["spec"]["dependsOn"]} == { + "hermes-observer-rbac", + "hermes", + } + assert "hermes-observer-bindings" not in { + item["name"] for item in hermes["spec"]["dependsOn"] + } + + +def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch): + module = _load() + seen = [] + monkeypatch.setattr( + module, + "_deny_sensitive_root", + lambda path, backup_name=None: seen.append((path, backup_name)), + ) + module._deny_sensitive_roots() + assert [name for _path, name in seen] == [ + "var-lib-rancher-k3s", + "var-lib-kubelet", + "run-k3s", + "run-containerd", + ] + assert len({name for _path, name in seen}) == 4 diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py new file mode 100644 index 00000000..3370fe4f --- /dev/null +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -0,0 +1,196 @@ +"""Host privilege-policy audits for the Hermes node SSH identity.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture + + +@pytest.mark.parametrize( + "database,record", + [ + ("group", "disk:x:6:atlas,hermes-agent\n"), + ("gshadow", "disk:!:hermes-agent:atlas\n"), + ("gshadow", "disk:!::atlas,hermes-agent\n"), + ], +) +def test_unexpected_group_or_gshadow_membership_fails_before_writes( + tmp_path: Path, monkeypatch, database: str, record: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + path = module.HOST_ETC / database + path.write_text( + "\n".join( + line + for line in originals[database].splitlines() + if not line.startswith("disk:") + ) + + "\n" + + record, + encoding="utf-8", + ) + before = {name: (module.HOST_ETC / name).read_bytes() for name in originals} + + with pytest.raises(module.HardeningError, match="group access"): + module.reconcile(public_key) + + assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before + + +@pytest.mark.parametrize( + "relative,value", + [ + ("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"), + ("sudoers", "ALL ALL=(ALL:ALL) ALL\n"), + ( + "polkit-1/rules.d/90-root.rules", + 'if (subject.user == "hermes-agent") return polkit.Result.YES;\n', + ), + ( + "polkit-1/rules.d/90-root.rules", + "unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n", + ), + ], +) +def test_sudo_and_root_equivalent_polkit_authority_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="sudo|polkit"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n") + with pytest.raises(module.HardeningError, match="external group"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize("database", ["passwd", "group", "initgroups", "shadow"]) +def test_external_identity_authority_sources_fail_closed( + tmp_path: Path, monkeypatch, database: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text( + f"passwd: files\n{database}: files ldap\n", encoding="utf-8" + ) + with pytest.raises(module.HardeningError, match="external group/account"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "value", + [ + "#1200 ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_IDS = #1200\nHERMES_IDS ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_GROUP = %#1200\nHERMES_GROUP ALL=(ALL:ALL) NOPASSWD: ALL\n", + ], +) +def test_visudo_valid_numeric_and_alias_grants_fail_closed( + tmp_path: Path, monkeypatch, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / "sudoers" + policy.write_text(value, encoding="utf-8") + validation = subprocess.run( + ["/usr/bin/visudo", "-c", "-f", str(policy)], + check=False, + capture_output=True, + text=True, + ) + assert validation.returncode == 0, validation.stderr + + with pytest.raises(module.HardeningError, match="sudo authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_sudo_include_is_bounded_to_the_audited_standard_directory( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("#includedir /opt/external-sudoers\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unaudited authority source"): + module.reconcile(public_key) + + sudoers.write_text("#includedir /etc/sudoers.d\n", encoding="utf-8") + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text().startswith(original) + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "relative,value", + [ + ( + "polkit-1/rules.d/90-hermes.rules", + "polkit.addRule(function(action, subject) {\n" + " if (subject.uid == 1200) return polkit.Result.YES;\n" + "});\n", + ), + ( + "polkit-1/localauthority/50-local.d/hermes.pkla", + "[Hermes]\nIdentity=unix-user:1200\n" + "Action=org.freedesktop.policykit.exec\nResultActive=yes\n", + ), + ], +) +def test_numeric_polkit_grants_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="polkit authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("root ALL=(ALL:ALL) ALL\n") + sudoers.chmod(0o666) + with pytest.raises(module.HardeningError, match="unsafe mutation"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) diff --git a/testing/tests/test_hermes_node_account_support.py b/testing/tests/test_hermes_node_account_support.py new file mode 100644 index 00000000..2d74d124 --- /dev/null +++ b/testing/tests/test_hermes_node_account_support.py @@ -0,0 +1,90 @@ +"""Shared fixtures for dedicated Hermes node-account contracts.""" + +from __future__ import annotations + +import base64 +import importlib.util +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py" +sys.path.insert(0, str(SCRIPT.parent)) + + +def _key(material: bytes, comment: str = "") -> str: + key_type = b"ssh-ed25519" + blob = len(key_type).to_bytes(4, "big") + key_type + material + suffix = f" {comment}" if comment else "" + return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix + + +def _typed_key(key_type: str, material: bytes, comment: str = "") -> str: + encoded_type = key_type.encode() + blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material + suffix = f" {comment}" if comment else "" + return f"{key_type} {base64.b64encode(blob).decode()}{suffix}" + + +def _load(): + spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fixture(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_home = tmp_path / "home" + host_etc.mkdir() + host_home.mkdir() + originals = { + "passwd": ( + "root:x:0:0:root:/root:/bin/bash\n" + "atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n" + "oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n" + ), + "group": ( + "root:x:0:\n" + "atlas:x:2000:\n" + "oceanus:x:2001:\n" + "disk:x:6:atlas\n" + "sudo:x:27:oceanus\n" + ), + "shadow": ( + "root:!:1:0:99999:7:::\n" + "atlas:!:1:0:99999:7:::\n" + "oceanus:!:1:0:99999:7:::\n" + ), + "gshadow": ( + "root:!::\n" + "atlas:!::\n" + "oceanus:!::\n" + "disk:!::atlas\n" + "sudo:!::oceanus\n" + ), + } + for name, value in originals.items(): + (host_etc / name).write_text(value, encoding="utf-8") + key = _key(b"synthetic-hermes-key") + other = _key(b"human-operator-key") + for user in ("atlas", "oceanus"): + ssh = host_home / user / ".ssh" + ssh.mkdir(parents=True) + (ssh / "authorized_keys").write_text( + f"{other} {user}\n{key}\n", encoding="utf-8" + ) + public_key = tmp_path / "public-key" + public_key.write_text(key + "\n", encoding="utf-8") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_HOME", host_home) + monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid()) + monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid()) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit") + monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None) + return module, originals, key, other, public_key diff --git a/testing/tests/test_hermes_node_acl_coverage.py b/testing/tests/test_hermes_node_acl_coverage.py new file mode 100644 index 00000000..572fd0d2 --- /dev/null +++ b/testing/tests/test_hermes_node_acl_coverage.py @@ -0,0 +1,206 @@ +"""Behavioral branch coverage for node-storage ACL hardening.""" + +from __future__ import annotations + +import errno +import os +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture, _load + + +def test_acl_decoder_rejects_version_permissions_and_adds_missing_mask(): + module = _load() + valid = [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + wrong_version = module.ACL_HEADER.pack(99) + b"".join( + module.ACL_ENTRY.pack(*entry) for entry in valid + ) + with pytest.raises(module.HardeningError, match="version"): + module._decode_acl(wrong_version, 0o755) + bad_permission = module._encode_acl([*valid, (module.ACL_USER, 8, 1200)]) + with pytest.raises(module.HardeningError, match="permission"): + module._decode_acl(bad_permission, 0o755) + updated = module._decode_acl( + module._acl_with_deny(module._encode_acl(valid), 0o755), 0o755 + ) + assert (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID) in updated + assert (module.ACL_USER, 0, module.ACCOUNT_UID) in updated + + +def test_read_acl_handles_absence_and_propagates_other_errors(monkeypatch, tmp_path): + module = _load() + path = tmp_path / "root" + path.mkdir() + + def missing(*_args, **_kwargs): + raise OSError(errno.ENODATA, "missing") + + monkeypatch.setattr(module.os, "getxattr", missing) + assert module._read_acl(path) == b"" + + def denied(*_args, **_kwargs): + raise OSError(errno.EPERM, "denied") + + monkeypatch.setattr(module.os, "getxattr", denied) + with pytest.raises(OSError): + module._read_acl(path) + + +@pytest.mark.parametrize("value", [b"", b"X", b"A", b"Nextra"]) +def test_acl_backup_validator_rejects_malformed_encodings(value): + module = _load() + with pytest.raises(module.HardeningError, match="backup is malformed"): + module._validate_acl_backup(value) + module._validate_acl_backup(b"N") + valid = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + module._validate_acl_backup(b"A" + valid) + + +def test_acl_backup_reuses_valid_existing_copy(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + root = host_etc / "hermes-node-boundary" + root.mkdir(parents=True) + backup = root / "k3s.acl" + backup.write_bytes(b"N") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + module._acl_backup_once(tmp_path / "k3s", b"") + assert backup.read_bytes() == b"N" + + +def test_acl_backup_short_write_cleans_temporary(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + monkeypatch.setattr(module.os, "write", lambda _fd, _value: 0) + with pytest.raises(module.HardeningError, match="short sensitive"): + module._acl_backup_once(tmp_path / "k3s", b"") + assert not list((host_etc / "hermes-node-boundary").glob(".*.hermes-*")) + + +def test_acl_backup_tolerates_concurrent_first_writer(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + real_link = os.link + + def race(source, target, **kwargs): + real_link(source, target, **kwargs) + raise FileExistsError() + + monkeypatch.setattr(module.os, "link", race) + module._acl_backup_once(tmp_path / "k3s", b"") + assert (host_etc / "hermes-node-boundary/k3s.acl").read_bytes() == b"N" + + +def test_sensitive_root_failure_restores_existing_acl(tmp_path, monkeypatch): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + current = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + reads = iter((current, current)) + writes = [] + monkeypatch.setattr(module, "_read_acl", lambda _path: next(reads)) + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, "setxattr", lambda *args, **kwargs: writes.append(args[2]) + ) + with pytest.raises(module.HardeningError, match="validation failed"): + module._deny_sensitive_root(root) + assert writes[-1] == current + + +@pytest.mark.parametrize("remove_errno", [errno.ENODATA, errno.EPERM]) +def test_sensitive_root_failure_removes_new_acl_or_propagates_cleanup_error( + tmp_path, monkeypatch, remove_errno +): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "_read_acl", lambda _path: b"") + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, + "setxattr", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("set failed")), + ) + monkeypatch.setattr( + module.os, + "removexattr", + lambda *_a, **_k: (_ for _ in ()).throw(OSError(remove_errno, "remove")), + ) + expected = OSError if remove_errno == errno.EPERM else RuntimeError + with pytest.raises(expected): + module._deny_sensitive_root(root) + + +def test_database_validation_failure_rolls_back_all_writes(tmp_path, monkeypatch): + module, originals, _key_value, _other, _public = _fixture(tmp_path, monkeypatch) + real_read = module._read_regular + calls = 0 + + def missing_expected(path, maximum=module.MAX_ACCOUNT_FILE): + nonlocal calls + calls += 1 + if calls == 1: + return originals["passwd"].encode(), path.stat(follow_symlinks=False) + return real_read(path, maximum) + + monkeypatch.setattr(module, "_read_regular", missing_expected) + with pytest.raises(module.HardeningError, match="validation failed"): + module._reconcile_databases() + for name, value in originals.items(): + assert (module.HOST_ETC / name).read_text() == value + + +def test_preexisting_target_key_removal_is_verified(tmp_path, monkeypatch): + module, _originals, key, _other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + "\n", encoding="utf-8") + real_write = module.atomic_write + + def ignore_preinstall_target(path, value, metadata): + if path == target and value == b"": + return + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", ignore_preinstall_target) + with pytest.raises(module.HardeningError, match="preexisting Hermes authorization"): + module._move_key(public) + + +def test_existing_target_without_key_does_not_need_precleanup(tmp_path, monkeypatch): + module, _originals, _key_value, other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(other + "\n", encoding="utf-8") + module._move_key(public) + assert target.read_text().endswith("\n") diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py new file mode 100644 index 00000000..8c011b5b --- /dev/null +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -0,0 +1,221 @@ +"""Behavioral branch coverage for dedicated node-account privilege audits.""" + +from __future__ import annotations + +import os +import struct +import sys +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_node_account_support import _load + + +def _audit_module(): + hardening = _load() + return sys.modules[hardening.audit_membership.__module__] + + +def test_member_parser_handles_empty_valid_malformed_and_duplicates(): + module = _audit_module() + assert module._members("", "group") == set() + assert module._members("one,two", "group") == {"one", "two"} + for value in ("one,", " one", "one,one"): + with pytest.raises(module.HardeningError): + module._members(value, "group") + + +def test_membership_audit_accepts_unrelated_records_and_rejects_account(): + module = _audit_module() + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "atlas"]], [["disk", "!", "atlas", ""]] + ) + with pytest.raises(module.HardeningError, match="supplementary"): + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "hermes-agent"]], [] + ) + with pytest.raises(module.HardeningError, match="gshadow"): + module.audit_membership("hermes-agent", [], [["disk", "!", "hermes-agent", ""]]) + + +@pytest.mark.parametrize("kind", ["mode", "symlink"]) +def test_policy_files_reject_unsafe_directories(tmp_path, kind): + module = _audit_module() + host_etc = tmp_path / "etc" + host_etc.mkdir() + root = host_etc / "sudoers.d" + if kind == "mode": + root.mkdir(mode=0o777) + root.chmod(0o777) + else: + target = tmp_path / "target" + target.mkdir() + root.symlink_to(target) + with pytest.raises(module.HardeningError, match="unsafe sudo/polkit"): + module._policy_files(host_etc, tmp_path / "share", os.getuid()) + + +def test_policy_file_count_is_bounded(tmp_path): + module = _audit_module() + root = tmp_path / "etc/sudoers.d" + root.mkdir(parents=True) + for index in range(257): + (root / str(index)).write_text("# safe\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="too many"): + module._policy_files(tmp_path / "etc", tmp_path / "share", os.getuid()) + + +def _acl(*entries): + return struct.pack(" bytes: + command = old + b" " + new + b" " + ref + b"\n" + framed = f"{len(command) + 4:04x}".encode() + command + return framed + (b"0000" if terminator else b"") + + +def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( + monkeypatch, +): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + with pytest.raises(broker.PolicyError, match="canonical ASCII"): + broker._validate_receive_pack( + _packet(zero, commit, b"refs/heads/hermes/\xff"), "sentinel" + ) + command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) + with pytest.raises(broker.PolicyError, match="terminator"): + broker._validate_receive_pack(command, "sentinel") + many = command * (broker.MAX_PUSH_COMMANDS + 1) + b"0000" + with pytest.raises(broker.PolicyError, match="too many"): + broker._validate_receive_pack(many, "sentinel") + + +def test_upstream_request_infers_byte_length_and_requires_stream_length(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"result", content_type="application/x-git-upload-pack-result") + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + == b"result" + ) + assert seen[0][0].get_header("Content-length") == "7" + with pytest.raises(broker.PolicyError, match="length is missing"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + + +def test_upstream_request_rejects_wrong_type_and_supports_getcode_status(): + broker = _load("scm_broker") + + class CodeOnly(Response): + def __init__(self, body, content_type): + super().__init__(body, content_type=content_type) + del self.status + + def getcode(self): + return 200 + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: CodeOnly( + b"advertisement", "application/x-git-upload-pack-advertisement" + ), + ) + == b"advertisement" + ) + with pytest.raises(broker.PolicyError, match="response type"): + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: Response(b"bad", content_type="text/plain"), + ) diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py new file mode 100644 index 00000000..25d94fa0 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -0,0 +1,321 @@ +"""RBAC, Flux, and human-review contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import json + +import pytest +import yaml + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + +def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool: + return any( + (api_group in rule["apiGroups"] or "*" in rule["apiGroups"]) + and (resource in rule["resources"] or "*" in rule["resources"]) + and (verb in rule["verbs"] or "*" in rule["verbs"]) + for rule in role["rules"] + ) + + +def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + ) + cluster_role, namespaced_role, binding = documents + assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2" + assert _can_i(cluster_role, "", "nodes", "get") + assert _can_i(namespaced_role, "", "pods", "get") + assert _can_i(namespaced_role, "", "pods/log", "get") + assert _can_i( + namespaced_role, + "kustomize.toolkit.fluxcd.io", + "kustomizations", + "list", + ) + + denied = ( + ("", "secrets", "get"), + ("", "serviceaccounts/token", "create"), + ("", "pods", "create"), + ("", "pods/exec", "create"), + ("", "pods/attach", "create"), + ("", "pods/portforward", "create"), + ("rbac.authorization.k8s.io", "clusterrolebindings", "create"), + ("authorization.k8s.io", "selfsubjectaccessreviews", "create"), + ("apps", "deployments", "patch"), + ) + assert all( + not _can_i(role, *request) + for request in denied + for role in (cluster_role, namespaced_role) + ) + + bindings = yaml.safe_load( + (ROOT / "services/hermes-observer-bindings/rolebindings.yaml").read_text( + encoding="utf-8" + ) + )["items"] + namespaces = {item["metadata"]["namespace"] for item in bindings} + assert "hermes-scm" not in namespaces + assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces + assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings) + + +def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent(): + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + assert "GIT_ASKPASS" not in json.dumps(agent) + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + assert broker["spec"]["template"]["spec"]["serviceAccountName"] == ( + "hermes-scm-broker" + ) + assert "gitea-token" in json.dumps(broker) + container = broker["spec"]["template"]["spec"]["containers"][0] + assert container["resources"]["limits"]["memory"] == "768Mi" + assert container["livenessProbe"]["failureThreshold"] == 10 + tmp = next( + item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp" + ) + assert tmp["emptyDir"]["sizeLimit"] == "1Gi" + + policy = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text( + encoding="utf-8" + ) + ) + ingress = policy["spec"]["ingress"] + assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == { + "kubernetes.io/metadata.name": "hermes" + } + assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-agent" + } + + vault = ( + ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault + assert '"hermes/developer-gitea" ""' in vault + agent_start = vault.index('write_policy_and_role "hermes-agent"') + agent_end = vault.index("write_policy_and_role", agent_start + 1) + assert "developer-gitea" not in vault[agent_start:agent_end] + + +def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle(): + hermes_resources = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + )["resources"] + assert "namespace.yaml" in hermes_resources + assert "scm-common" in hermes_resources + + observer = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "dependsOn" not in observer["spec"] + observer_objects = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + if item + ] + assert {item["kind"] for item in observer_objects} <= { + "ClusterRole", + "ClusterRoleBinding", + } + + hermes_flux = yaml.safe_load( + ( + ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "hermes-observer-rbac" in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + bindings_flux = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert {item["name"] for item in bindings_flux["spec"]["dependsOn"]} == { + "hermes", + "hermes-observer-rbac", + } + assert bindings_flux["metadata"]["name"] not in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + + applications = ( + ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml" + ).read_text(encoding="utf-8") + assert "hermes-scm-agent-code" not in applications + + broker_code = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert broker_code["spec"]["path"] == "./services/hermes/scm-common" + assert broker_code["spec"]["targetNamespace"] == "hermes-scm" + assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}] + + +def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift(): + script = ( + ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh" + ).read_text(encoding="utf-8") + + assert 'required_approvals\\\":1' in script + assert 'enable_push_whitelist\\\":true' in script + assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert 'enable_merge_whitelist\\\":true' in script + assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert "protection differs from the human-review policy" in script + assert "api_request PATCH" not in script + job = yaml.safe_load( + (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text() + ) + init = job["spec"]["template"]["spec"]["initContainers"][0] + assert init["image"] == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert init["securityContext"]["runAsNonRoot"] is True + assert init["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert "/opt/python/bin/python3" in script + + +def _protection(rule_name: str, priority: int, helper, **overrides): + value = { + "rule_name": rule_name, + "priority": priority, + "created_at": "2026-01-01T00:00:00Z", + **helper._required("bstein"), + } + value.update(overrides) + return value + + +def test_branch_protection_uses_first_effective_glob_by_priority(): + helper = _load_path( + "branch_protection_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + earlier_drift = _protection("m*", 1, helper, required_approvals=0) + later_exact = _protection("main", 2, helper) + + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + + earlier_drift["required_approvals"] = 1 + assert ( + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + == "PRESENT" + ) + brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0) + with pytest.raises(helper.PolicyError, match="effective master protection differs"): + helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein") + + upper_glob = _protection("M*", 1, helper, required_approvals=0) + assert ( + helper.evaluate(json.dumps([upper_glob, later_exact]).encode(), "main", "bstein") + == "PRESENT" + ) + + +def test_branch_protection_matches_gitea_v1238_tie_order(): + helper = _load_path( + "branch_protection_order_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + older_glob = _protection( + "m*", 1, helper, created_at="2025-01-01T00:00:00Z", required_approvals=0 + ) + newer_plain = _protection( + "MAIN", 1, helper, created_at="2026-01-01T00:00:00Z" + ) + assert ( + helper.evaluate(json.dumps([older_glob, newer_plain]).encode(), "main", "bstein") + == "PRESENT" + ) + + older_glob["created_at"] = "2026-01-02T00:00:00Z" + earlier_glob = _protection( + "m{ain,aster}", + 1, + helper, + created_at="2026-01-01T00:00:00Z", + required_approvals=0, + ) + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([older_glob, earlier_glob]).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + ("rule_name", "branch", "expected"), + [ + ("release/*", "release/v1.17", True), + ("release/**/v1.17", "release/test/1/v1.17", True), + ("release/*/v1.17", "release/test/1/v1.17", False), + ("*", "release/v1.16", False), + ("**", "release/v1.16", True), + ("MAIN", "main", True), + ("M*", "main", False), + ("m{ain,aster}", "master", True), + (r"m\ain", "main", True), + ], +) +def test_branch_glob_matches_gitea_v1238_source_corpus( + rule_name: str, branch: str, expected: bool +): + helper = _load_path( + "branch_protection_glob_parity_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + assert helper._matches(rule_name, branch) is expected + + +@pytest.mark.parametrize( + "rules", + [ + [{"rule_name": "main", "priority": 0, "created_at": "2026-01-01T00:00:00Z"}], + [{"rule_name": "main", "priority": 1, "created_at": "not-a-time"}], + [{"rule_name": "main"}], + ], +) +def test_branch_protection_rejects_ambiguous_rules_before_creation(rules): + helper = _load_path( + "branch_protection_adversarial_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + with pytest.raises(helper.PolicyError): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_branch_protection_reports_absent_only_when_no_rule_matches(): + helper = _load_path( + "branch_protection_absent_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection("release/*", 1, helper)] + assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py new file mode 100644 index 00000000..2fa9683f --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -0,0 +1,237 @@ +"""Bounded I/O and concurrency contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import io +import json +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_agent_broker_client_sends_no_credential_or_authorization_header(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"{}", content_type="application/json") + + assert client.read("/api/v1/repos/atlas/cassandra", opener=opener) == b"{}" + request = seen[0][0] + assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata" + assert request.get_header("Authorization") is None + assert json.loads(request.data) == {"path": "/api/v1/repos/atlas/cassandra"} + + +@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"]) +def test_broker_content_length_is_canonical_and_bounded(raw: str): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = raw + + with pytest.raises(broker.PolicyError, match="request length"): + broker._content_length(headers, broker.MAX_GIT_REQUEST) + + +def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries(): + broker = _load("scm_broker") + token = "runtime-sentinel" + safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1) + inbound_timeouts = [] + spool, length = broker._spool_bounded( + io.BytesIO(safe), + broker.MAX_GIT_REQUEST, + len(safe), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + set_timeout=inbound_timeouts.append, + ) + try: + assert length == len(safe) + assert spool._rolled is True + assert spool.read(4) == b"xxxx" + assert inbound_timeouts + assert all(0 < value <= 30 for value in inbound_timeouts) + finally: + spool.close() + + crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._spool_bounded( + io.BytesIO(crossing), + broker.MAX_GIT_REQUEST, + len(crossing), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + ) + + +def test_large_upstream_response_is_streamed_through_bounded_spool(): + broker = _load("scm_broker") + body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1) + seen = [] + + def opener(request, **_kwargs): + seen.append(request) + assert request.get_header("Content-length") == "7" + assert request.data.read() == b"request" + return Response(body, content_type="application/x-git-upload-pack-result") + + streamed = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + body_length=7, + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + stream_result=True, + opener=opener, + ) + spool, length = streamed + try: + assert length == len(body) + assert spool._rolled is True + assert spool.read(3) == b"zzz" + assert len(seen) == 1 + finally: + spool.close() + + +def test_body_deadline_fails_closed_before_reading(): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="deadline"): + broker._spool_bounded( + io.BytesIO(b"body"), + broker.MAX_GIT_REQUEST, + 4, + token="runtime-sentinel", + context="Git request", + deadline=broker.time.monotonic() - 1, + ) + + +def test_upstream_stream_applies_absolute_socket_deadline(): + broker = _load("scm_broker") + + class DeadlineSocket: + def __init__(self): + self.values = [] + + def settimeout(self, value): + self.values.append(value) + + response = Response(b"safe", content_type="application/x-git-upload-pack-result") + sock = DeadlineSocket() + response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() + spool, length = broker._spool_response( + response, broker.MAX_GIT_RESPONSE, "sentinel" + ) + try: + assert length == 4 + assert sock.values + assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values) + finally: + spool.close() + + +def test_broker_rejects_work_when_concurrency_slots_are_exhausted(): + broker = _load("scm_broker") + + class FakeSocket: + def __init__(self): + self.value = b"" + + def sendall(self, value): + self.value += value + + def shutdown(self, _how): + return None + + def close(self): + return None + + server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler) + try: + assert broker.BoundedThreadingHTTPServer.__module__ == "scm_broker_server" + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + request = FakeSocket() + server.process_request(request, ("127.0.0.1", 1)) + assert b"503 Service Unavailable" in request.value + finally: + server._slots.release() + server._slots.release() + server._slots.release() + server.server_close() + + +def test_three_simultaneous_broker_calls_have_bounded_capacity(): + server_module = _load("scm_broker_server") + semaphore = server_module.threading.BoundedSemaphore( + server_module.MAX_CONCURRENT_REQUESTS + ) + assert server_module.MAX_CONCURRENT_REQUESTS >= 3 + assert [semaphore.acquire(blocking=False) for _ in range(3)] == [True, True, True] + assert semaphore.acquire(blocking=False) is False + + +def test_header_deadline_is_absolute_against_slow_trickle(monkeypatch): + server_module = _load("scm_broker_server") + + class Stream: + def __init__(self): + self.value = io.BytesIO(b"GET /healthz HTTP/1.1\r\n") + + def read(self, length): + return self.value.read(length) + + class Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + ticks = iter((0.0, 0.02, 0.04, 0.06, 0.08, 0.11)) + monkeypatch.setattr(server_module.time, "monotonic", lambda: next(ticks)) + connection = Connection() + reader = server_module._AbsoluteDeadlineReader(Stream(), connection) + reader.begin(0.1) + + with pytest.raises(TimeoutError, match="absolute SCM header deadline"): + reader.readline(65537) + + assert connection.timeouts == pytest.approx([0.08, 0.06, 0.04, 0.02]) + + +def test_broker_bounds_header_count_size_and_duplicate_lengths(): + broker = _load("scm_broker") + + def validate(headers): + handler = object.__new__(broker.BrokerHandler) + handler.headers = headers + handler._validate_headers() + + too_many = Message() + for index in range(broker.MAX_HEADERS + 1): + too_many[f"X-Test-{index}"] = "safe" + with pytest.raises(broker.PolicyError, match="too many headers"): + validate(too_many) + + too_large = Message() + too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES + with pytest.raises(broker.PolicyError, match="safe limit"): + validate(too_large) + + duplicate = Message() + duplicate["Content-Length"] = "1" + duplicate["Content-Length"] = "1" + with pytest.raises(broker.PolicyError, match="duplicate Content-Length"): + validate(duplicate) diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py new file mode 100644 index 00000000..6e2184d9 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -0,0 +1,55 @@ +"""Shared fixtures for credential-isolated Hermes SCM broker tests.""" + +from __future__ import annotations + +import importlib.util +import io +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + path = SCRIPTS / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"test_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Response: + def __init__(self, body: bytes, *, status: int = 200, content_type: str): + self.body = body + self.stream = io.BytesIO(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.stream.read(limit) + + +def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes: + command = old + b" " + new + b" " + ref + b"\x00report-status\n" + return f"{len(command) + 4:04x}".encode() + command + b"0000PACK" diff --git a/testing/tests/test_hermes_scm_server_coverage.py b/testing/tests/test_hermes_scm_server_coverage.py new file mode 100644 index 00000000..7a358640 --- /dev/null +++ b/testing/tests/test_hermes_scm_server_coverage.py @@ -0,0 +1,180 @@ +"""Behavioral branch coverage for the SCM broker HTTP server primitives.""" + +from __future__ import annotations + +import io + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _FlushBuffer(io.BytesIO): + def __init__(self): + super().__init__() + self.flushes = 0 + + def flush(self): + self.flushes += 1 + + +def test_absolute_reader_supports_idle_limit_newline_eof_and_attribute_delegation(): + module = _load("scm_broker_server") + stream = io.BytesIO(b"first\nsecond") + reader = module._AbsoluteDeadlineReader(stream, _Connection()) + assert reader.readline(3) == b"fir" + reader.begin(5) + assert reader.readline() == b"st\n" + assert reader.readline() == b"second" + assert reader.readline() == b"" + assert reader.closed is False + reader.end() + assert reader.readline() == b"" + + +def _handler_type(module): + class StubHandler: + request_version = "HTTP/1.1" + command = "GET" + + def setup(self): + self.connection = _Connection() + self.rfile = io.BytesIO(b"") + self.wfile = _FlushBuffer() + + def parse_request(self): + return self.parse_result + + def send_error(self, code, *_args): + self.errors.append(code) + + def do_GET(self): + self.calls.append("GET") + + return type( + "DeadlineHandler", (module.AbsoluteHeaderDeadlineMixin, StubHandler), {} + ) + + +def _handler(module, raw: bytes, *, parse=True): + handler_type = _handler_type(module) + handler = object.__new__(handler_type) + handler.setup() + handler._header_reader._stream = io.BytesIO(raw) + handler.parse_result = parse + handler.errors = [] + handler.calls = [] + return handler + + +def test_header_mixin_setup_and_known_request_dispatch(): + module = _load("scm_broker_server") + handler = _handler(module, b"GET /healthz HTTP/1.1\r\n") + handler.handle_one_request() + assert handler.calls == ["GET"] + assert handler.wfile.flushes == 1 + assert handler._header_reader._deadline is None + + +@pytest.mark.parametrize( + ("raw", "parse", "command", "expected_error", "closed"), + [ + (b"x" * 65537, True, "GET", 414, False), + (b"", True, "GET", None, True), + (b"GET / HTTP/1.1\r\n", False, "GET", None, False), + (b"TRACE / HTTP/1.1\r\n", True, "TRACE", 501, False), + ], +) +def test_header_mixin_rejects_long_empty_unparsed_and_unknown_requests( + raw, parse, command, expected_error, closed +): + module = _load("scm_broker_server") + handler = _handler(module, raw, parse=parse) + handler.command = command + handler.close_connection = False + handler.handle_one_request() + assert handler.errors == ([] if expected_error is None else [expected_error]) + assert handler.close_connection is closed + + +def test_header_mixin_closes_on_absolute_timeout(monkeypatch): + module = _load("scm_broker_server") + handler = _handler(module, b"GET / HTTP/1.1\r\n") + handler.close_connection = False + monkeypatch.setattr( + handler._header_reader, + "readline", + lambda *_a, **_k: (_ for _ in ()).throw(TimeoutError()), + ) + handler.handle_one_request() + assert handler.close_connection is True + + +class _Slots: + def __init__(self, available=True): + self.available = available + self.releases = 0 + + def acquire(self, *, blocking): + assert blocking is False + return self.available + + def release(self): + self.releases += 1 + + +def _server_shell(module, available=True): + server = object.__new__(module.BoundedThreadingHTTPServer) + server._slots = _Slots(available) + server.shutdowns = [] + server.shutdown_request = server.shutdowns.append + return server + + +def test_bounded_server_delegates_accepted_request(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + calls = [] + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda self, request, address: calls.append((request, address)), + ) + server.process_request("socket", ("127.0.0.1", 1)) + assert calls == [("socket", ("127.0.0.1", 1))] + assert server._slots.releases == 0 + + +def test_bounded_server_releases_on_dispatch_failure(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("dispatch")), + ) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 + assert server.shutdowns == ["socket"] + + +def test_bounded_server_thread_always_releases(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request_thread", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("handler")), + ) + with pytest.raises(RuntimeError, match="handler"): + server.process_request_thread("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..fc22a8ab 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -26,8 +31,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path - - + + @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,73 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_branch_floor_and_requires_branch_data( + tmp_path: Path, +) -> None: + """Tracked modules must report and meet the configured branch floor.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + { + "coverage": { + "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + + +def test_run_check_limits_branch_floor_to_explicit_branch_tracked_files( + tmp_path: Path, +) -> None: + """A branch rollout can cover selected security modules without hiding lines.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["new.py", "legacy.py"], + "branch_tracked_files": ["new.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == [] From 73fefbb5d92ef07371a9a997f22d848acd2d06ed Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 08:16:35 -0300 Subject: [PATCH 12/46] hermes: gate mixed image lane APIs --- dockerfiles/Dockerfile.hermes-agent | 10 +- .../Dockerfile.hermes-agent.dockerignore | 1 + .../hermes_lane_compatibility_regression.py | 120 +++++++ services/hermes/kustomization.yaml | 1 + .../hermes/scripts/cli_lane_capabilities.py | 130 ++++++++ services/hermes/scripts/cli_lane_dispatch.py | 24 +- services/hermes/scripts/cli_lane_evidence.py | 7 +- services/hermes/scripts/cli_lane_execution.py | 22 +- services/hermes/scripts/cli_lane_files.py | 5 +- .../hermes/scripts/cli_lane_finalization.py | 7 + services/hermes/scripts/cli_lane_provider.py | 17 +- .../hermes/scripts/cli_lane_quarantine.py | 5 +- services/hermes/scripts/cli_lane_runner.py | 8 + testing/quality_contract.json | 53 ++++ testing/tests/conftest.py | 26 ++ testing/tests/test_hermes_agent_layout.py | 29 ++ testing/tests/test_hermes_cli_capabilities.py | 272 ++++++++++++++++ .../tests/test_hermes_cli_dispatch_runtime.py | 197 ++++++++++++ .../tests/test_hermes_cli_evidence_edges.py | 194 ++++++++++++ .../tests/test_hermes_cli_execution_edges.py | 218 +++++++++++++ .../test_hermes_cli_finalization_edges.py | 164 ++++++++++ .../test_hermes_cli_foundation_coverage.py | 207 ++++++++++++ .../tests/test_hermes_cli_provider_edges.py | 173 ++++++++++ .../tests/test_hermes_cli_quarantine_edges.py | 185 +++++++++++ .../tests/test_hermes_cli_records_edges.py | 230 ++++++++++++++ .../tests/test_hermes_cli_recovery_edges.py | 298 ++++++++++++++++++ .../tests/test_hermes_cli_retention_edges.py | 89 ++++++ 27 files changed, 2655 insertions(+), 37 deletions(-) create mode 100644 dockerfiles/hermes_lane_compatibility_regression.py create mode 100644 services/hermes/scripts/cli_lane_capabilities.py create mode 100644 testing/tests/conftest.py create mode 100644 testing/tests/test_hermes_cli_capabilities.py create mode 100644 testing/tests/test_hermes_cli_dispatch_runtime.py create mode 100644 testing/tests/test_hermes_cli_evidence_edges.py create mode 100644 testing/tests/test_hermes_cli_execution_edges.py create mode 100644 testing/tests/test_hermes_cli_finalization_edges.py create mode 100644 testing/tests/test_hermes_cli_foundation_coverage.py create mode 100644 testing/tests/test_hermes_cli_provider_edges.py create mode 100644 testing/tests/test_hermes_cli_quarantine_edges.py create mode 100644 testing/tests/test_hermes_cli_records_edges.py create mode 100644 testing/tests/test_hermes_cli_recovery_edges.py create mode 100644 testing/tests/test_hermes_cli_retention_edges.py diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index e0024e44..1416aae7 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -1552,8 +1552,15 @@ COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decompos COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py +COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/ -RUN /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \ +RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \ + HERMES_COMPATIBILITY_MODE=legacy \ + /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \ + && /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \ + && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \ + HERMES_COMPATIBILITY_MODE=patched \ + /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \ && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \ /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \ && rm /tmp/patch-hermes-execution-safety.py \ @@ -1564,6 +1571,7 @@ RUN /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \ /tmp/hermes_execution_regression_support.py \ /tmp/hermes_run_safety_regression.py \ /tmp/hermes_decomposition_safety_regression.py \ + /tmp/hermes_lane_compatibility_regression.py \ && find /tmp/hermes-lane-regression -depth -delete COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index e6cb3493..0abc8045 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -13,6 +13,7 @@ !dockerfiles/hermes_execution_regression_support.py !dockerfiles/hermes_run_safety_regression.py !dockerfiles/hermes_decomposition_safety_regression.py +!dockerfiles/hermes_lane_compatibility_regression.py !services/ !services/hermes/ !services/hermes/scripts/ diff --git a/dockerfiles/hermes_lane_compatibility_regression.py b/dockerfiles/hermes_lane_compatibility_regression.py new file mode 100644 index 00000000..7ed69c63 --- /dev/null +++ b/dockerfiles/hermes_lane_compatibility_regression.py @@ -0,0 +1,120 @@ +"""Exercise mounted CLI scripts against the exact image Kanban API.""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +_HOME = tempfile.TemporaryDirectory(prefix="hermes-lane-compatibility-") +os.environ["HERMES_HOME"] = _HOME.name +os.environ["HERMES_CLI_LANE_CONCURRENCY"] = "1" +lane_source = Path(os.environ["HERMES_CLI_LANE_SOURCE"]) +sys.path.insert(0, str(lane_source)) + +from cli_lane_capabilities import ( # noqa: E402 + detect_kanban_capabilities, + initialize_kanban_capabilities, + runtime_health, +) +from cli_lane_config import TerminalIdentity # noqa: E402 +from cli_lane_dispatch import recover_orphans # noqa: E402 +from cli_lane_finalization import _finalize_document_db # noqa: E402 +from hermes_cli import kanban_db # noqa: E402 + + +class MountedLaneCompatibilityTests(unittest.TestCase): + """Verify safe behavior before and after the image API patch.""" + + def setUp(self) -> None: + self.connection = kanban_db.connect() + + def tearDown(self) -> None: + self.connection.close() + + @staticmethod + def _document(summary: str) -> dict: + return { + "result": json.dumps( + { + "status": "completed", + "summary": summary, + "changed_files": [], + "tests_run": ["mixed source compatibility"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + sort_keys=True, + ), + "summary": summary, + "metadata": {"matrix": os.environ["HERMES_COMPATIBILITY_MODE"]}, + } + + def test_exact_image_api_matrix(self) -> None: + mode = os.environ["HERMES_COMPATIBILITY_MODE"] + capabilities = initialize_kanban_capabilities(kanban_db) + self.assertTrue(capabilities.exact_run_completion) + self.assertEqual(capabilities.ready, mode == "patched") + self.assertEqual(runtime_health()["state"], "ready" if mode == "patched" else "deferred") + self.assertLess(Path(_HOME.name, "cli-lanes/runtime-health.json").stat().st_size, 2048) + + def test_current_active_run_completion_stays_safe(self) -> None: + task_id = kanban_db.create_task(self.connection, title="active accepted result") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + outcome = _finalize_document_db( + kanban_db, + TerminalIdentity("default", task_id, run_id, "pending"), + self._document("active result"), + ) + self.assertEqual(outcome, "committed") + self.assertEqual(kanban_db.get_task(self.connection, task_id).status, "done") + + def test_ended_run_replay_is_enabled_only_after_patch(self) -> None: + mode = os.environ["HERMES_COMPATIBILITY_MODE"] + task_id = kanban_db.create_task(self.connection, title="ended accepted result") + self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) + run_id = kanban_db.get_task(self.connection, task_id).current_run_id + self.assertTrue( + kanban_db.block_task( + self.connection, + task_id, + reason="simulate post-journal error", + expected_run_id=run_id, + ) + ) + outcome = _finalize_document_db( + kanban_db, + TerminalIdentity("default", task_id, run_id, "pending"), + self._document("ended result"), + ) + self.assertEqual(outcome, "committed" if mode == "patched" else "deferred") + + def test_old_reclaim_api_is_never_called_unguarded(self) -> None: + capabilities = detect_kanban_capabilities(kanban_db) + if capabilities.exact_run_reclaim: + self.skipTest("patched API exercises guarded reclaim in the exact-run suite") + with mock.patch.object( + kanban_db, + "reclaim_task", + side_effect=AssertionError("legacy unguarded reclaim called"), + ), mock.patch("cli_lane_dispatch.recover_pending_finalizations", return_value=0): + recover_orphans() + + +def main() -> int: + """Run the compatibility matrix for the selected exact image source.""" + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(MountedLaneCompatibilityTests) + ) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 706a56b0..9c8df595 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -62,6 +62,7 @@ configMapGenerator: - claude=scripts/claude - claude_command_policy.py=scripts/claude_command_policy.py - cli_lane_board.py=scripts/cli_lane_board.py + - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py - cli_lane_config.py=scripts/cli_lane_config.py - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py - cli_lane_evidence.py=scripts/cli_lane_evidence.py diff --git a/services/hermes/scripts/cli_lane_capabilities.py b/services/hermes/scripts/cli_lane_capabilities.py new file mode 100644 index 00000000..c54c8bb0 --- /dev/null +++ b/services/hermes/scripts/cli_lane_capabilities.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Detect and publish the Kanban API boundary required by CLI lanes.""" + +from __future__ import annotations + +import inspect +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable + +from cli_lane_config import STATE_ROOT, utc_now +from cli_lane_files import atomic_json, load_json + + +_CAPABILITIES: KanbanCapabilities | None = None +_CAPABILITY_SOURCE: tuple[int, int] | None = None + + +@dataclass(frozen=True) +class KanbanCapabilities: + """Safety-relevant keyword support exposed by the image's Kanban DB.""" + + exact_run_completion: bool + ended_run_replay: bool + exact_run_reclaim: bool + + @property + def ready(self) -> bool: + """Return whether dispatch and every recovery transition are safe.""" + return ( + self.exact_run_completion + and self.ended_run_replay + and self.exact_run_reclaim + ) + + @property + def deferred_features(self) -> tuple[str, ...]: + """List fixed, bounded capability names absent from the image API.""" + missing = [] + if not self.exact_run_completion: + missing.append("exact-run-completion") + if not self.ended_run_replay: + missing.append("ended-run-replay") + if not self.exact_run_reclaim: + missing.append("exact-run-reclaim") + return tuple(missing) + + +def _explicit_keyword(function: Callable[..., Any], keyword: str) -> bool: + """Require an explicit keyword parameter rather than trusting ``**kwargs``.""" + try: + parameter = inspect.signature(function).parameters.get(keyword) + except (TypeError, ValueError): + return False + return parameter is not None and parameter.kind in { + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + } + + +def detect_kanban_capabilities(kanban_db: Any) -> KanbanCapabilities: + """Inspect the loaded image API without invoking a mutating DB operation.""" + complete_task = getattr(kanban_db, "complete_task", None) + reclaim_task = getattr(kanban_db, "reclaim_task", None) + return KanbanCapabilities( + exact_run_completion=callable(complete_task) + and _explicit_keyword(complete_task, "expected_run_id"), + ended_run_replay=callable(complete_task) + and _explicit_keyword(complete_task, "replay_ended_run_id"), + exact_run_reclaim=callable(reclaim_task) + and _explicit_keyword(reclaim_task, "expected_run_id"), + ) + + +def _source_identity(kanban_db: Any) -> tuple[int, int]: + """Bind cached capability evidence to the two inspected callables.""" + return ( + id(getattr(kanban_db, "complete_task", None)), + id(getattr(kanban_db, "reclaim_task", None)), + ) + + +def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]: + """Return a bounded operator-readable compatibility health document.""" + return { + "component": "cli-lane-runner", + "state": "ready" if capabilities.ready else "deferred", + "ready": capabilities.ready, + "active_run_completion_safe": capabilities.exact_run_completion, + "deferred_features": list(capabilities.deferred_features), + "capabilities": asdict(capabilities), + "observed_at": utc_now(), + } + + +def initialize_kanban_capabilities( + kanban_db: Any, + *, + health_path: Path | None = None, +) -> KanbanCapabilities: + """Detect startup compatibility and durably expose ready/deferred health.""" + global _CAPABILITIES, _CAPABILITY_SOURCE + capabilities = detect_kanban_capabilities(kanban_db) + _CAPABILITIES = capabilities + _CAPABILITY_SOURCE = _source_identity(kanban_db) + destination = health_path or STATE_ROOT / "runtime-health.json" + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_json(destination, _health_document(capabilities), 0o600) + if not capabilities.ready: + print( + "CLI lane safety compatibility deferred; missing image APIs: " + + ", ".join(capabilities.deferred_features), + file=sys.stderr, + flush=True, + ) + return capabilities + + +def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities: + """Return startup evidence, redetecting only when the loaded API changes.""" + if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE: + return initialize_kanban_capabilities(kanban_db) + return _CAPABILITIES + + +def runtime_health(path: Path | None = None) -> dict[str, Any]: + """Read the bounded health document for probes and operator diagnostics.""" + document = load_json(path or STATE_ROOT / "runtime-health.json") + return document if isinstance(document, dict) else {} diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 7d386a8c..8a740dbf 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -10,6 +10,7 @@ import time from typing import Any from cli_lane_board import _external, _record_board_access_error, _task_value +from cli_lane_capabilities import initialize_kanban_capabilities, kanban_capabilities from cli_lane_config import ( BOARD_CORRUPTION_ERRORS, DEFAULT_CLAIM_TTL, @@ -41,6 +42,8 @@ def recover_orphans() -> None: from hermes_cli import kanban_db recover_pending_finalizations() + if not kanban_capabilities(kanban_db).exact_run_reclaim: + return try: boards = kanban_db.list_boards(include_archived=False) except Exception as error: @@ -97,10 +100,14 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str for task in tasks: task_id = str(_task_value(task, "id", "")) assignee = str(_task_value(task, "assignee", "") or "") - if task_id and not assignee and str(_task_value(task, "status", "")) == "ready": - if kanban_db.assign_task(conn, task_id, "cli-auto"): - task = kanban_db.get_task(conn, task_id) - assignee = "cli-auto" + if ( + task_id + and not assignee + and str(_task_value(task, "status", "")) == "ready" + and kanban_db.assign_task(conn, task_id, "cli-auto") + ): + task = kanban_db.get_task(conn, task_id) + assignee = "cli-auto" if ( not task_id or (board, task_id) in active @@ -129,8 +136,11 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str def main() -> int: """Continuously bridge external Kanban lanes to provider CLIs.""" + from hermes_cli import kanban_db + RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644) + capabilities = initialize_kanban_capabilities(kanban_db) recover_orphans() workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8)) futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {} @@ -147,7 +157,11 @@ def main() -> int: maybe_gc_lane_artifacts() active = set(futures.values()) try: - newly_claimed = claim_ready(active, workers - len(futures)) + newly_claimed = ( + claim_ready(active, workers - len(futures)) + if capabilities.ready + else [] + ) BOARD_CORRUPTION_ERRORS.pop("board-registry", None) except Exception as error: _record_board_access_error("board-registry", error) diff --git a/services/hermes/scripts/cli_lane_evidence.py b/services/hermes/scripts/cli_lane_evidence.py index f261e870..d85abb0e 100644 --- a/services/hermes/scripts/cli_lane_evidence.py +++ b/services/hermes/scripts/cli_lane_evidence.py @@ -4,6 +4,7 @@ from __future__ import annotations import ctypes +import contextlib import errno import hashlib import json @@ -75,7 +76,7 @@ def _retire_terminal_entry( (authority_name or path.name).encode("utf-8") ).hexdigest()[:16] source_digest = hashlib.sha256( - f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode("utf-8") + f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode() ).hexdigest()[:16] for sequence in range(32): staging = f".retire.{path_digest}.{source_digest}.{sequence}" @@ -157,15 +158,13 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: if descriptor is not None: os.close(descriptor) if temporary_stat is not None: - try: + with contextlib.suppress(OSError): _retire_terminal_entry( Path(temporary), temporary_stat, board_descriptor=directory, quarantine_descriptor=directory, ) - except OSError: - pass os.close(directory) def _terminal_document_digest(document: dict[str, Any]) -> str: diff --git a/services/hermes/scripts/cli_lane_execution.py b/services/hermes/scripts/cli_lane_execution.py index 3b24b468..46a3d7b0 100644 --- a/services/hermes/scripts/cli_lane_execution.py +++ b/services/hermes/scripts/cli_lane_execution.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib import json import sqlite3 import time @@ -98,7 +99,7 @@ def execute_claim(board: str, task_id: str) -> None: return True def comment(body: str) -> None: - try: + with contextlib.suppress(OSError, sqlite3.Error): _board_call( kanban_db, board, @@ -109,10 +110,8 @@ def execute_claim(board: str, task_id: str) -> None: body, ), ) - except (OSError, sqlite3.Error): - # Route state is also written to the durable lane-state file; - # a later heartbeat or terminal result remains authoritative. - pass + # Route state is also written to the durable lane-state file; + # a later heartbeat or terminal result remains authoritative. try: previous_route = state.get("current_route") @@ -395,15 +394,20 @@ def execute_claim(board: str, task_id: str) -> None: reason = reason or str(structured.get("summary") or "") if reason is None: reason = result.output[-4000:] + failure_reason = reason or ( + f"{route.provider} worker failed with exit {result.returncode}" + ) + failure_kind = ( + "transient" if result.capacity_failure else "capability" + ) _board_call( kanban_db, board, - lambda fresh: kanban_db.block_task( + lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task( fresh, task_id, - reason=reason - or f"{route.provider} worker failed with exit {result.returncode}", - kind="transient" if result.capacity_failure else "capability", + reason=failure_reason, + kind=failure_kind, expected_run_id=run_id, ), ) diff --git a/services/hermes/scripts/cli_lane_files.py b/services/hermes/scripts/cli_lane_files.py index 9d160da0..64422a1e 100644 --- a/services/hermes/scripts/cli_lane_files.py +++ b/services/hermes/scripts/cli_lane_files.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib import json import os import re @@ -36,10 +37,8 @@ def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: finally: if descriptor is not None: os.close(descriptor) - try: + with contextlib.suppress(FileNotFoundError): temporary.unlink() - except FileNotFoundError: - pass def _fsync_directory(directory: Path) -> None: """Persist directory-entry changes after an atomic rename or quarantine.""" diff --git a/services/hermes/scripts/cli_lane_finalization.py b/services/hermes/scripts/cli_lane_finalization.py index 5599f830..ef39ffc5 100644 --- a/services/hermes/scripts/cli_lane_finalization.py +++ b/services/hermes/scripts/cli_lane_finalization.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any from cli_lane_board import _board_call, _external, _record_board_access_error, _task_value +from cli_lane_capabilities import kanban_capabilities from cli_lane_config import ( TerminalIdentity, TerminalRecoverySnapshot, @@ -39,6 +40,8 @@ def _recover_exact_run( """Make an exact external run retryable after journal recovery fails.""" if identity is None or canonical_run_id(identity.run_id) is None: return False + if not kanban_capabilities(kanban_db).exact_run_reclaim: + return False def operation(conn: Any) -> bool: task = kanban_db.get_task(conn, identity.task_id) @@ -178,11 +181,15 @@ def _finalize_document_db( current_run_id = _task_value(task, "current_run_id", None) completion_guard: dict[str, int] if status == "running" and current_run_id == identity.run_id: + if not kanban_capabilities(kanban_db).exact_run_completion: + return "deferred" completion_guard = {"expected_run_id": identity.run_id} elif status in {"ready", "blocked", "triage"} and current_run_id is None: # The journal may have survived an older post-persistence error # path that ended its run. The patched DB verifies atomically that # this is still the latest ended run before allowing completion. + if not kanban_capabilities(kanban_db).ended_run_replay: + return "deferred" completion_guard = {"replay_ended_run_id": identity.run_id} else: return "stale" diff --git a/services/hermes/scripts/cli_lane_provider.py b/services/hermes/scripts/cli_lane_provider.py index 53a05bba..07a3d02b 100644 --- a/services/hermes/scripts/cli_lane_provider.py +++ b/services/hermes/scripts/cli_lane_provider.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib import json import os import selectors @@ -166,17 +167,13 @@ def _signal_worker_tree( if _process_identity_matches(pid, start_time): groups.add(process_group) for process_group in groups: - try: + with contextlib.suppress(ProcessLookupError): os.killpg(process_group, sig) - except ProcessLookupError: - pass for pid, (_, start_time) in descendants.items(): if not _process_identity_matches(pid, start_time): continue - try: + with contextlib.suppress(ProcessLookupError): os.kill(pid, sig) - except ProcessLookupError: - pass def _terminate_worker_process( process: subprocess.Popen[str], @@ -187,10 +184,8 @@ def _terminate_worker_process( descendants.update(_descendant_processes(process.pid)) if process.poll() is None: _signal_worker_tree(process.pid, descendants, signal.SIGTERM) - try: + with contextlib.suppress(subprocess.TimeoutExpired): process.wait(timeout=10) - except subprocess.TimeoutExpired: - pass # Provider terminal tools create their own sessions, so killing only the # native CLI's process group leaves test/build subprocesses orphaned. The @@ -357,10 +352,8 @@ def run_provider( file_result = load_json(result_file) if file_result.get("status") in cli_lane_goal.RESULT_STATUSES: result.structured = file_result - try: + with contextlib.suppress(OSError): result_file.chmod(0o600) - except OSError: - pass return result atomic_json(state_file, state) diff --git a/services/hermes/scripts/cli_lane_quarantine.py b/services/hermes/scripts/cli_lane_quarantine.py index 21dfddb1..8c599184 100644 --- a/services/hermes/scripts/cli_lane_quarantine.py +++ b/services/hermes/scripts/cli_lane_quarantine.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib import hashlib import json import os @@ -136,10 +137,8 @@ def _quarantine_terminal( quarantine_descriptor = os.dup(board_descriptor) quarantine_dir = board_dir else: - try: + with contextlib.suppress(FileExistsError): os.mkdir("quarantine", 0o700, dir_fd=board_descriptor) - except FileExistsError: - pass quarantine_descriptor = os.open( "quarantine", board_flags, diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 5a51e928..ed32e3b7 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -12,6 +12,14 @@ import subprocess import uuid import cli_lane_goal +from cli_lane_capabilities import ( + KanbanCapabilities, + _explicit_keyword, + detect_kanban_capabilities, + initialize_kanban_capabilities, + kanban_capabilities, + runtime_health, +) from cli_lane_board import ( _board_call, _external, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..cfcc0e43 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -20,6 +20,23 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/cli_lane_board.py", + "services/hermes/scripts/cli_lane_capabilities.py", + "services/hermes/scripts/cli_lane_config.py", + "services/hermes/scripts/cli_lane_dispatch.py", + "services/hermes/scripts/cli_lane_evidence.py", + "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_files.py", + "services/hermes/scripts/cli_lane_finalization.py", + "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_prompt.py", + "services/hermes/scripts/cli_lane_provider.py", + "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_records.py", + "services/hermes/scripts/cli_lane_recovery.py", + "services/hermes/scripts/cli_lane_retention.py", + "services/hermes/scripts/cli_lane_routing.py", + "services/hermes/scripts/cli_lane_runner.py", "testing/__init__.py", "testing/quality_contract.py", "testing/quality_docs.py", @@ -43,6 +60,23 @@ "services/comms/scripts/tests", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/cli_lane_board.py", + "services/hermes/scripts/cli_lane_capabilities.py", + "services/hermes/scripts/cli_lane_config.py", + "services/hermes/scripts/cli_lane_dispatch.py", + "services/hermes/scripts/cli_lane_evidence.py", + "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_files.py", + "services/hermes/scripts/cli_lane_finalization.py", + "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_prompt.py", + "services/hermes/scripts/cli_lane_provider.py", + "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_records.py", + "services/hermes/scripts/cli_lane_recovery.py", + "services/hermes/scripts/cli_lane_retention.py", + "services/hermes/scripts/cli_lane_routing.py", + "services/hermes/scripts/cli_lane_runner.py", "testing/tests", "testing" ], @@ -58,6 +92,7 @@ "coverage_sources": [ "ci/scripts", "scripts.render.dashboards_render_atlas", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -124,6 +159,7 @@ "ci/tests/**/*.py", "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", + "services/hermes/scripts/cli_lane_*.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py" ], @@ -174,6 +210,23 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/cli_lane_board.py", + "services/hermes/scripts/cli_lane_capabilities.py", + "services/hermes/scripts/cli_lane_config.py", + "services/hermes/scripts/cli_lane_dispatch.py", + "services/hermes/scripts/cli_lane_evidence.py", + "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_files.py", + "services/hermes/scripts/cli_lane_finalization.py", + "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_prompt.py", + "services/hermes/scripts/cli_lane_provider.py", + "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_records.py", + "services/hermes/scripts/cli_lane_recovery.py", + "services/hermes/scripts/cli_lane_retention.py", + "services/hermes/scripts/cli_lane_routing.py", + "services/hermes/scripts/cli_lane_runner.py", "testing/quality_contract.py", "testing/quality_docs.py", "testing/quality_hygiene.py", diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py new file mode 100644 index 00000000..09dcb0a1 --- /dev/null +++ b/testing/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared pytest boundaries for repository automation tests.""" + +from __future__ import annotations + +import sys + +import pytest + + +@pytest.fixture(autouse=True) +def _patched_hermes_cli_db_contract(request, monkeypatch): + """Keep legacy lane test doubles focused on their original safety boundary.""" + module_name = request.module.__name__.rsplit(".", 1)[-1] + if not module_name.startswith("test_hermes_cli_") or module_name.endswith( + "capabilities" + ): + return + capability_module = sys.modules.get("cli_lane_capabilities") + if capability_module is None: + return + safe = capability_module.KanbanCapabilities(True, True, True) + for name, module in tuple(sys.modules.items()): + if ( + name == "cli_lane_runner" or name.startswith("cli_lane_") + ) and hasattr(module, "kanban_capabilities"): + monkeypatch.setattr(module, "kanban_capabilities", lambda _db: safe) diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index f5bd5074..2e2ef65e 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -308,6 +308,35 @@ def test_cli_lane_domain_modules_are_all_mounted_with_the_runner(): assert expected <= mounted +def test_cli_lane_config_refresh_does_not_restart_active_work(): + """Keep projected script refreshes separate from pod lifecycle changes.""" + manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text()) + coordinator = next( + item + for item in manifest["configMapGenerator"] + if item["name"] == "hermes-coordinator" + ) + deployment = _agent_deployment() + lane = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "cli-lane-runner" + ) + mount = next( + item for item in lane["volumeMounts"] if item["name"] == "coordinator" + ) + + assert coordinator["options"]["disableNameSuffixHash"] is True + assert mount == { + "name": "coordinator", + "mountPath": "/opt/coordinator", + "readOnly": True, + } + assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get( + "metadata", {} + ).get("annotations", {}) + + def test_agent_dashboard_reconnects_all_transient_websockets(): dockerfile = ( HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" diff --git a/testing/tests/test_hermes_cli_capabilities.py b/testing/tests/test_hermes_cli_capabilities.py new file mode 100644 index 00000000..adc6e3ca --- /dev/null +++ b/testing/tests/test_hermes_cli_capabilities.py @@ -0,0 +1,272 @@ +"""Mixed-image capability gates for mounted Hermes CLI lane scripts.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_support import _completed_result, lanes + + +def _old_complete( + _conn, + _task_id, + *, + result=None, + summary=None, + metadata=None, + expected_run_id=None, +): + return bool(result or summary or metadata or expected_run_id) + + +def _new_complete( + _conn, + _task_id, + *, + result=None, + summary=None, + metadata=None, + expected_run_id=None, + replay_ended_run_id=None, +): + return bool(result or summary or metadata or expected_run_id or replay_ended_run_id) + + +def _old_reclaim(_conn, _task_id, *, reason, signal_fn=None): + return bool(reason or signal_fn) + + +def _new_reclaim( + _conn, + _task_id, + *, + reason, + signal_fn=None, + expected_run_id=None, +): + return bool(reason or signal_fn or expected_run_id) + + +def _db(*, patched: bool): + return SimpleNamespace( + complete_task=_new_complete if patched else _old_complete, + reclaim_task=_new_reclaim if patched else _old_reclaim, + ) + + +def _terminal_document(run_id: int = 7) -> dict: + structured = _completed_result("accepted result") + return { + "board": "cassandra", + "task_id": "t_compat", + "expected_run_id": run_id, + "result": json.dumps(structured, sort_keys=True), + "summary": "accepted result", + "metadata": {}, + "kanban_state": "pending", + "recorded_at": lanes.utc_now(), + } + + +def test_capability_detection_requires_explicit_safety_keywords(): + old = lanes.detect_kanban_capabilities(_db(patched=False)) + new = lanes.detect_kanban_capabilities(_db(patched=True)) + + assert old.exact_run_completion is True + assert old.ended_run_replay is False + assert old.exact_run_reclaim is False + assert old.ready is False + assert old.deferred_features == ("ended-run-replay", "exact-run-reclaim") + assert new.ready is True + assert new.deferred_features == () + + variadic = SimpleNamespace( + complete_task=lambda *_args, **_kwargs: True, + reclaim_task=lambda *_args, **_kwargs: True, + ) + assert lanes.detect_kanban_capabilities(variadic).ready is False + assert lanes._explicit_keyword(object(), "expected_run_id") is False + + +def test_startup_health_moves_from_bounded_deferred_to_ready( + tmp_path: Path, + capsys, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + health = tmp_path / "runtime-health.json" + old = lanes.initialize_kanban_capabilities(_db(patched=False), health_path=health) + old_document = lanes.runtime_health(health) + + assert old.ready is False + assert old_document["state"] == "deferred" + assert old_document["ready"] is False + assert old_document["active_run_completion_safe"] is True + assert old_document["deferred_features"] == [ + "ended-run-replay", + "exact-run-reclaim", + ] + assert health.stat().st_size < 2048 + assert health.stat().st_mode & 0o777 == 0o600 + assert "compatibility deferred" in capsys.readouterr().err + + new_db = _db(patched=True) + new = lanes.kanban_capabilities(new_db) + lanes.initialize_kanban_capabilities(new_db, health_path=health) + assert new.ready is True + assert lanes.runtime_health(health)["state"] == "ready" + + +@pytest.mark.parametrize( + ("status", "run_id", "expected"), + [("running", 7, "committed"), ("blocked", None, "deferred")], +) +def test_old_image_completes_only_the_exact_active_run( + tmp_path: Path, + monkeypatch, + status: str, + run_id: int | None, + expected: str, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + task = SimpleNamespace( + id="t_compat", + status=status, + current_run_id=run_id, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + calls = [] + + def complete( + _conn, + _task_id, + *, + result=None, + summary=None, + metadata=None, + expected_run_id=None, + ): + calls.append(expected_run_id) + task.status = "done" + task.current_run_id = None + task.completed_run_id = expected_run_id + task.result = result + return True + + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + get_task=lambda _conn, _task_id: task, + complete_task=complete, + reclaim_task=_old_reclaim, + ) + identity = lanes.TerminalIdentity("cassandra", "t_compat", 7, "pending") + + outcome = lanes._finalize_document_db(db, identity, _terminal_document()) + + assert outcome == expected + assert calls == ([7] if status == "running" else []) + if expected == "deferred": + assert task.status == "blocked" + + +def test_old_image_preserves_pending_and_prepared_journals( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + task = SimpleNamespace( + id="t_compat", + status="blocked", + current_run_id=None, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + get_task=lambda _conn, _task_id: task, + complete_task=_old_complete, + reclaim_task=_old_reclaim, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + pending, _record = lanes._write_terminal_record( + lanes.state_path("cassandra", "t_compat"), + board="cassandra", + task_id="t_compat", + run_id=7, + structured=_completed_result("accepted result"), + summary="accepted result", + metadata={}, + ) + + assert lanes.recover_pending_finalizations() == 0 + assert pending.exists() + assert len(list(pending.parent.glob("*.terminal.prepared-*.json"))) == 1 + assert task.status == "blocked" + + +def test_old_image_never_calls_unguarded_reclaim(tmp_path: Path, monkeypatch): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + task = SimpleNamespace( + id="t_running", + status="running", + current_run_id=11, + assignee="cli-auto", + ) + calls = [] + + def reclaim(_conn, _task_id, *, reason, signal_fn=None): + calls.append((reason, signal_fn)) + return True + + db = SimpleNamespace( + complete_task=_old_complete, + reclaim_task=reclaim, + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + list_tasks=lambda _conn: [task], + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + + lanes.recover_orphans() + + assert calls == [] + + +def test_main_detects_capabilities_before_startup_recovery( + tmp_path: Path, + monkeypatch, +): + events = [] + db = _db(patched=False) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "result.schema.json") + monkeypatch.setattr( + lanes, + "initialize_kanban_capabilities", + lambda candidate: events.append(("detect", candidate)) + or lanes.KanbanCapabilities(True, False, False), + ) + + def stop_after_detection(): + events.append(("recover", db)) + raise RuntimeError("stop after startup boundary") + + monkeypatch.setattr(lanes, "recover_orphans", stop_after_detection) + + with pytest.raises(RuntimeError, match="startup boundary"): + lanes.main() + + assert events == [("detect", db), ("recover", db)] diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py new file mode 100644 index 00000000..74684105 --- /dev/null +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -0,0 +1,197 @@ +"""Dispatcher startup, degraded-board, and worker-loop regressions.""" + +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_support import lanes + + +def test_board_slug_and_connection_failure_paths(monkeypatch): + assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra" + assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback" + failures = [] + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: failures.append((board, str(error))), + ) + db = SimpleNamespace( + connect=lambda **_kwargs: (_ for _ in ()).throw(OSError("offline")) + ) + assert lanes._connect_healthy_board(db, "broken") is None + assert failures == [("broken", "offline")] + + +def test_orphan_recovery_bounds_registry_and_per_board_failures( + monkeypatch, +): + failures = [] + registry_db = SimpleNamespace( + complete_task=lambda *_args, expected_run_id=None, **_kwargs: True, + reclaim_task=lambda *_args, expected_run_id=None, **_kwargs: True, + list_boards=lambda **_kwargs: (_ for _ in ()).throw(OSError("registry")), + ) + monkeypatch.setitem( + sys.modules, + "hermes_cli", + SimpleNamespace(kanban_db=registry_db), + ) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: failures.append((board, str(error))), + ) + lanes.recover_orphans() + assert failures == [("board-registry", "registry")] + + class Connection: + def close(self): + failures.append(("close", "yes")) + + board_db = SimpleNamespace( + complete_task=registry_db.complete_task, + reclaim_task=registry_db.reclaim_task, + list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "broken"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + list_tasks=lambda _conn: (_ for _ in ()).throw(OSError("board read")), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board_db)) + lanes.recover_orphans() + assert ("broken", "board read") in failures + assert ("close", "yes") in failures + + +def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatch): + monkeypatch.setitem( + sys.modules, + "hermes_cli", + SimpleNamespace(kanban_db=SimpleNamespace()), + ) + assert lanes.claim_ready(set(), 0) == [] + + class Connection: + def close(self): + return None + + task = SimpleNamespace( + id="t_claim", + status="ready", + assignee="cli-auto", + ) + db = SimpleNamespace( + list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("claim raced") + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + assert lanes.claim_ready(set(), 1) == [] + + +class _Future: + def __init__(self): + self.polls = 0 + + def done(self): + self.polls += 1 + return self.polls >= 1 + + def result(self): + raise RuntimeError("worker failed") + + +class _Pool: + def __init__(self, max_workers): + self.max_workers = max_workers + self.future = _Future() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def submit(self, function, board, task_id): + assert callable(function) + assert (board, task_id) == ("cassandra", "t_loop") + return self.future + + +def test_ready_dispatch_loop_submits_and_reaps_failed_workers( + tmp_path: Path, + monkeypatch, + capsys, +): + db = SimpleNamespace() + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") + monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + monkeypatch.setattr( + lanes, + "initialize_kanban_capabilities", + lambda _db: lanes.KanbanCapabilities(True, True, True), + ) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) + claims = [[("cassandra", "t_loop")], []] + monkeypatch.setattr(lanes, "claim_ready", lambda *_args: claims.pop(0)) + monkeypatch.setattr( + lanes.concurrent.futures, + "ThreadPoolExecutor", + _Pool, + ) + sleeps = [] + + def stop_second_loop(_seconds): + sleeps.append(True) + if len(sleeps) == 2: + raise RuntimeError("stop loop") + + monkeypatch.setattr(lanes.time, "sleep", stop_second_loop) + + with pytest.raises(RuntimeError, match="stop loop"): + lanes.main() + + assert "worker future failed: worker failed" in capsys.readouterr().err + + +def test_deferred_dispatch_health_never_claims_new_work( + tmp_path: Path, + monkeypatch, +): + db = SimpleNamespace() + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") + monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + monkeypatch.setattr( + lanes, + "initialize_kanban_capabilities", + lambda _db: lanes.KanbanCapabilities(True, False, False), + ) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) + monkeypatch.setattr( + lanes, + "claim_ready", + lambda *_args: pytest.fail("deferred startup must not claim work"), + ) + monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _Pool) + monkeypatch.setattr( + lanes.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")), + ) + with pytest.raises(RuntimeError, match="stop loop"): + lanes.main() diff --git a/testing/tests/test_hermes_cli_evidence_edges.py b/testing/tests/test_hermes_cli_evidence_edges.py new file mode 100644 index 00000000..c28ff43f --- /dev/null +++ b/testing/tests/test_hermes_cli_evidence_edges.py @@ -0,0 +1,194 @@ +"""Failure-boundary coverage for immutable lane evidence and retention.""" + +from __future__ import annotations + +import errno +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_support import _pending_terminal_record, lanes + + +def test_rename_noreplace_rejects_missing_kernel_support(monkeypatch): + monkeypatch.setattr(lanes.ctypes, "CDLL", lambda *_args, **_kwargs: object()) + with pytest.raises(OSError) as raised: + lanes._rename_noreplace("a", "b", source_dir=1, destination_dir=1) + assert raised.value.errno == errno.ENOSYS + + +def test_rename_noreplace_surfaces_kernel_errno(monkeypatch): + class Rename: + argtypes = None + restype = None + + def __call__(self, *_args): + return -1 + + monkeypatch.setattr( + lanes.ctypes, + "CDLL", + lambda *_args, **_kwargs: SimpleNamespace(renameat2=Rename()), + ) + monkeypatch.setattr(lanes.ctypes, "get_errno", lambda: errno.EEXIST) + with pytest.raises(FileExistsError): + lanes._rename_noreplace("a", "b", source_dir=1, destination_dir=1) + + +def test_retirement_reports_missing_replacement_collision_and_restoration( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "pending" + path.write_text("old", encoding="utf-8") + source = path.stat() + directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + path.unlink() + assert lanes._retire_terminal_entry( + path, + source, + board_descriptor=directory, + quarantine_descriptor=directory, + ) == "missing" + + path.write_text("new", encoding="utf-8") + assert lanes._retire_terminal_entry( + path, + source, + board_descriptor=directory, + quarantine_descriptor=directory, + ) == "replacement" + + replacement = path.stat() + monkeypatch.setattr( + lanes, + "_rename_noreplace", + lambda *_args, **_kwargs: (_ for _ in ()).throw(FileExistsError()), + ) + assert lanes._retire_terminal_entry( + path, + replacement, + board_descriptor=directory, + quarantine_descriptor=directory, + ) == "collision" + finally: + os.close(directory) + + +def test_retirement_preserves_a_postcheck_replacement(tmp_path: Path, monkeypatch): + path = tmp_path / "pending" + path.write_text("old", encoding="utf-8") + source = path.stat() + directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + calls = [] + + def race(source_name, destination, **_kwargs): + calls.append((source_name, destination)) + if len(calls) == 1: + os.rename(source_name, destination, src_dir_fd=directory, dst_dir_fd=directory) + staged = tmp_path / destination + staged.unlink() + staged.write_text("replacement", encoding="utf-8") + else: + os.rename(source_name, destination, src_dir_fd=directory, dst_dir_fd=directory) + + monkeypatch.setattr(lanes, "_rename_noreplace", race) + try: + assert lanes._retire_terminal_entry( + path, + source, + board_descriptor=directory, + quarantine_descriptor=directory, + ) == "replacement" + finally: + os.close(directory) + assert path.read_text(encoding="utf-8") == "replacement" + + +def test_evidence_writer_bounds_payload_and_cleans_failed_temporary( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 8) + with pytest.raises(ValueError, match="bounded"): + lanes._write_json_noreplace(tmp_path / "large.json", {"value": "x" * 5000}) + + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 1024 * 1024) + real_retire = lanes._retire_terminal_entry + monkeypatch.setattr( + lanes, + "_rename_noreplace", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("rename failed")), + ) + monkeypatch.setattr( + lanes, + "_retire_terminal_entry", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("cleanup failed")), + ) + with pytest.raises(OSError, match="rename failed"): + lanes._write_json_noreplace(tmp_path / "failed.json", {"value": "small"}) + monkeypatch.setattr(lanes, "_retire_terminal_entry", real_retire) + + +def test_evidence_paths_and_collision_validation_fail_closed( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + identity = lanes.TerminalIdentity("cassandra", "t_evidence", 9, "pending") + document = _pending_terminal_record("cassandra", "t_evidence", 9, "accepted") + with pytest.raises(ValueError, match="evidence state"): + lanes._terminal_evidence_path(identity, "pending", document) + + monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False) + monkeypatch.setattr(lanes, "_terminal_evidence_identity", lambda _path: None) + with pytest.raises(OSError, match="identity is invalid"): + lanes._persist_prepared_evidence(identity, document) + with pytest.raises(OSError, match="conflict evidence identity"): + lanes._persist_conflict_evidence(identity, document, "loser") + + +def test_existing_evidence_with_different_result_is_rejected( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + identity = lanes.TerminalIdentity("cassandra", "t_evidence", 10, "pending") + document = _pending_terminal_record("cassandra", "t_evidence", 10, "accepted") + prepared_identity = lanes.TerminalIdentity("cassandra", "t_evidence", 10, "prepared") + conflict_identity = lanes.TerminalIdentity("cassandra", "t_evidence", 10, "conflict") + monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False) + monkeypatch.setattr( + lanes, + "_terminal_evidence_identity", + lambda path: conflict_identity if "conflict" in path.name else prepared_identity, + ) + monkeypatch.setattr(lanes, "_load_small_json", lambda _path: {}) + with pytest.raises(OSError, match="conflicting result"): + lanes._persist_prepared_evidence(identity, document) + with pytest.raises(OSError, match="different result"): + lanes._persist_conflict_evidence(identity, document, "loser") + + +def test_retention_failure_paths_are_bounded(tmp_path: Path, monkeypatch, capsys): + path = tmp_path / "artifact" + path.write_text("data", encoding="utf-8") + observed = path.stat() + monkeypatch.setattr( + lanes, + "_retire_terminal_entry", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("busy")), + ) + assert lanes._unlink_artifact_if_same(path, observed) is False + + monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0) + monkeypatch.setattr( + lanes, + "gc_lane_artifacts", + lambda **_kwargs: (_ for _ in ()).throw(OSError("volume")), + ) + assert lanes.maybe_gc_lane_artifacts(now=1000) == 0 + assert "retention deferred" in capsys.readouterr().err diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py new file mode 100644 index 00000000..07b9035d --- /dev/null +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -0,0 +1,218 @@ +"""Execution-loop edge coverage for preparation, callbacks, and recovery.""" + +from __future__ import annotations + +import sqlite3 +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +from testing.tests.test_hermes_cli_support import _completed_result, lanes + + +class _Connection: + def close(self): + return None + + +def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch): + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + lanes.execute_claim("cassandra", "missing") + + +def test_transient_callback_storage_errors_do_not_kill_provider( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_callbacks", + status="running", + current_run_id=20, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "exercise callbacks", + heartbeat_worker=lambda *_args, **_kwargs: (_ for _ in ()).throw( + sqlite3.OperationalError("heartbeat volume") + ), + add_comment=lambda *_args, **_kwargs: (_ for _ in ()).throw( + sqlite3.OperationalError("comment volume") + ), + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: "claude") + route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) + + def provider(*args, **_kwargs): + assert args[6]("still working") is True + return lanes.ProcessResult(1, "failed", None, False) + + monkeypatch.setattr(lanes, "run_provider", provider) + lanes.execute_claim("cassandra", "t_callbacks") + + assert blocks and blocks[0]["kind"] == "capability" + + +def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch): + task = SimpleNamespace( + id="t_restart", + status="running", + current_run_id=21, + assignee="cli-auto", + max_runtime_seconds=60, + ) + state_root = tmp_path / "lanes" + monkeypatch.setattr(lanes, "STATE_ROOT", state_root) + state_file = lanes.state_path("cassandra", "t_restart") + lanes.atomic_json(state_file, {"current_route": {"provider": "claude"}}) + blocks = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "missing.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume", + add_comment=lambda *_args, **_kwargs: None, + heartbeat_worker=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) + handoffs = [] + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: handoffs.append(output) or "handoff", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(1, "failed", None, False), + ) + + lanes.execute_claim("cassandra", "t_restart") + + assert handoffs == ["Previous provider log was unavailable after restart."] + assert blocks + + +def test_unreplayable_persistence_error_recovers_exact_run( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_persist", + status="running", + current_run_id=22, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "finish", + add_comment=lambda _conn, _task, _author, body: comments.append(body), + heartbeat_worker=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt", "high", "p", "c", "r", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "", _completed_result("accepted"), False + ), + ) + monkeypatch.setattr( + lanes, + "_write_terminal_record", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk")), + ) + monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False) + recovered = [] + monkeypatch.setattr( + lanes, + "_recover_exact_run", + lambda *args: recovered.append(args) or True, + ) + + lanes.execute_claim("cassandra", "t_persist") + + assert recovered and recovered[0][1].run_id == 22 + assert any("terminal replayable=False" in body for body in comments) + + +def test_unexpected_route_exception_blocks_the_exact_run( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_route", + status="running", + current_run_id=23, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "route", + add_comment=lambda *_args, **_kwargs: None, + heartbeat_worker=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("router down")), + ) + + lanes.execute_claim("cassandra", "t_route") + + assert blocks[0]["expected_run_id"] == 23 + assert "router down" in blocks[0]["reason"] diff --git a/testing/tests/test_hermes_cli_finalization_edges.py b/testing/tests/test_hermes_cli_finalization_edges.py new file mode 100644 index 00000000..0f2c2a2c --- /dev/null +++ b/testing/tests/test_hermes_cli_finalization_edges.py @@ -0,0 +1,164 @@ +"""Exact-run finalization guard and convergence edge coverage.""" + +from __future__ import annotations + +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +from testing.tests.test_hermes_cli_support import ( + _pending_terminal_record, + lanes, +) + + +class _Connection: + def close(self): + return None + + +def _db(task): + return SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + reclaim_task=lambda *_args, expected_run_id=None, **_kwargs: True, + complete_task=lambda *_args, expected_run_id=None, + replay_ended_run_id=None, **_kwargs: False, + ) + + +def test_exact_recovery_rejects_missing_capability_and_nonauthoritative_tasks( + monkeypatch, +): + identity = lanes.TerminalIdentity("cassandra", "t_recover", 3, "pending") + assert lanes._recover_exact_run(_db(None), None, "missing") is False + + monkeypatch.setattr( + lanes, + "kanban_capabilities", + lambda _db: lanes.KanbanCapabilities(True, True, False), + ) + assert lanes._recover_exact_run(_db(None), identity, "old image") is False + + monkeypatch.setattr( + lanes, + "kanban_capabilities", + lambda _db: lanes.KanbanCapabilities(True, True, True), + ) + assert lanes._recover_exact_run(_db(None), identity, "missing task") is False + internal = SimpleNamespace( + status="running", + current_run_id=3, + assignee="internal", + ) + assert lanes._recover_exact_run(_db(internal), identity, "internal") is False + + +def test_exact_recovery_bounds_board_access_failure(monkeypatch): + identity = lanes.TerminalIdentity("cassandra", "t_recover", 4, "pending") + db = _db(SimpleNamespace(status="running", current_run_id=4, assignee="cli-auto")) + db.connect = lambda **_kwargs: (_ for _ in ()).throw(OSError("volume")) + errors = [] + monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1) + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: errors.append((board, str(error))), + ) + assert lanes._recover_exact_run(db, identity, "storage") is False + assert errors[-1] == ("cassandra", "volume") + + +def test_finalizer_guards_invalid_runtime_identity_and_missing_task(monkeypatch): + identity = object.__new__(lanes.TerminalIdentity) + object.__setattr__(identity, "board", "cassandra") + object.__setattr__(identity, "task_id", "t_invalid") + object.__setattr__(identity, "run_id", 0) + object.__setattr__(identity, "state", "pending") + assert lanes._finalize_document_db(_db(None), identity, {}) == "stale" + + valid = lanes.TerminalIdentity("cassandra", "missing", 5, "pending") + assert lanes._finalize_document_db(_db(None), valid, {}) == "stale" + + running = SimpleNamespace( + status="running", + current_run_id=5, + completed_run_id=None, + result=None, + ) + monkeypatch.setattr( + lanes, + "kanban_capabilities", + lambda _db: lanes.KanbanCapabilities(False, False, False), + ) + assert lanes._finalize_document_db(_db(running), valid, {}) == "deferred" + + +class _Snapshot: + def __init__(self, document): + self.document = document + self.closed = 0 + + def close(self): + self.closed += 1 + + +def test_resolve_pending_rejects_invalid_document_and_unknown_retirement( + tmp_path: Path, + monkeypatch, +): + identity = lanes.TerminalIdentity("cassandra", "t_pending", 6, "pending") + invalid = _Snapshot({}) + monkeypatch.setattr(lanes, "_open_terminal_snapshot", lambda *_args: invalid) + assert lanes._resolve_pending_after_winner(tmp_path / "pending", identity, {}) is False + assert invalid.closed == 1 + + valid_document = _pending_terminal_record("cassandra", "t_pending", 6, "winner") + unknown = _Snapshot(valid_document) + monkeypatch.setattr(lanes, "_open_terminal_snapshot", lambda *_args: unknown) + monkeypatch.setattr(lanes, "_retire_snapshot_after_db", lambda *_args: "deferred") + assert ( + lanes._resolve_pending_after_winner( + tmp_path / "pending", + identity, + valid_document, + ) + is False + ) + assert unknown.closed == 1 + + +def test_resolve_pending_stops_after_bounded_replacement_churn( + tmp_path: Path, + monkeypatch, +): + identity = lanes.TerminalIdentity("cassandra", "t_churn", 7, "pending") + document = _pending_terminal_record("cassandra", "t_churn", 7, "winner") + snapshots = [] + + def snapshot(*_args): + value = _Snapshot(document) + snapshots.append(value) + return value + + monkeypatch.setattr(lanes, "_open_terminal_snapshot", snapshot) + monkeypatch.setattr(lanes, "_retire_snapshot_after_db", lambda *_args: "replacement") + assert lanes._resolve_pending_after_winner(tmp_path / "pending", identity, document) is False + assert len(snapshots) == 8 + assert all(item.closed == 1 for item in snapshots) + + +def test_terminal_finalizer_closes_supplied_invalid_snapshots( + tmp_path: Path, + monkeypatch, +): + snapshot = _Snapshot({}) + assert lanes._finalize_terminal_record(_db(None), tmp_path / "bad", snapshot=snapshot) == "invalid" + assert snapshot.closed == 1 + + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + path = lanes._terminal_path(lanes.state_path("board", "task"), 8) + empty = _Snapshot(None) + assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid" + assert empty.closed == 1 diff --git a/testing/tests/test_hermes_cli_foundation_coverage.py b/testing/tests/test_hermes_cli_foundation_coverage.py new file mode 100644 index 00000000..71f0c44a --- /dev/null +++ b/testing/tests/test_hermes_cli_foundation_coverage.py @@ -0,0 +1,207 @@ +"""Adversarial branch coverage for CLI lane foundation helpers.""" + +from __future__ import annotations + +import json +import sqlite3 +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace +from urllib.error import URLError + +import pytest + +from testing.tests.test_hermes_cli_support import _completed_result, lanes + + +def test_board_context_serializes_objects_and_storage_failure_is_bounded( + monkeypatch, +): + db = SimpleNamespace(build_worker_context=lambda _conn, _task: {"priority": 3}) + assert lanes._task_context(db, object(), "t_ctx") == '{\n "priority": 3\n}' + + connections = [] + + class Connection: + def close(self): + connections.append("closed") + + failing = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + ) + monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 2) + monkeypatch.setattr(lanes.time, "sleep", lambda _seconds: None) + with pytest.raises(sqlite3.OperationalError, match="volume unavailable"): + lanes._board_call( + failing, + "cassandra", + lambda _conn: (_ for _ in ()).throw( + sqlite3.OperationalError("volume unavailable") + ), + ) + assert connections == ["closed", "closed"] + + +def test_atomic_json_closes_an_unwrapped_descriptor_on_open_failure( + tmp_path: Path, + monkeypatch, +): + destination = tmp_path / "state.json" + real_fdopen = lanes.os.fdopen + descriptors = [] + + def fail_fdopen(descriptor, *_args, **_kwargs): + descriptors.append(descriptor) + raise OSError("cannot wrap descriptor") + + monkeypatch.setattr(lanes.os, "fdopen", fail_fdopen) + with pytest.raises(OSError, match="cannot wrap"): + lanes.atomic_json(destination, {"safe": True}) + monkeypatch.setattr(lanes.os, "fdopen", real_fdopen) + + assert descriptors + with pytest.raises(OSError): + lanes.os.fstat(descriptors[0]) + assert list(tmp_path.glob("*.tmp")) == [] + + +def test_json_and_terminal_path_helpers_fail_closed(tmp_path: Path, monkeypatch): + invalid = tmp_path / "invalid.json" + invalid.write_bytes(b"\xff") + assert lanes.load_json(invalid) == {} + array = tmp_path / "array.json" + array.write_text("[]", encoding="utf-8") + assert lanes.load_json(array) == {} + + with pytest.raises(ValueError, match="journal state"): + lanes._terminal_path(tmp_path / "task.json", 1, "prepared") + with pytest.raises(ValueError, match="SQLite"): + lanes._terminal_path(tmp_path / "task.json", True) + + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + outside = tmp_path / "outside.run-1.terminal.pending.json" + assert lanes._terminal_identity(outside) is None + nested = lanes.STATE_ROOT / "board" / "nested" / "x.json" + assert lanes._terminal_identity(nested) is None + missing = lanes.STATE_ROOT / "board" / "t.run-1.terminal.pending.json" + assert lanes._terminal_identity(missing) is None + assert lanes._terminal_evidence_identity(outside) is None + assert lanes._terminal_evidence_identity(nested) is None + missing_evidence = lanes.STATE_ROOT / "board" / ( + "t.run-1.terminal.prepared-" + "a" * 32 + ".json" + ) + assert lanes._terminal_evidence_identity(missing_evidence) is None + + +def test_goal_helpers_cover_compaction_and_deterministic_failures(): + compacted = lanes.cli_lane_goal._bounded("a" * 100, 40) + assert len(compacted) == 40 + assert "compacted" in compacted + assert lanes.cli_lane_goal.unfinished_result_reason( + {"status": "incomplete", "summary": ""} + ) == "worker explicitly reported incomplete work" + assert lanes.cli_lane_goal.unfinished_result_reason( + {"status": "blocked", "summary": "waiting"} + ) is None + assert "blockers" in lanes.cli_lane_goal.unfinished_result_reason( + {"status": "completed", "summary": "done", "blockers": ["remote"]} + ) + assert "unfinished" in lanes.cli_lane_goal.unfinished_result_reason( + { + "status": "completed", + "summary": "verification is pending", + "blockers": [], + "tests_run": [], + } + ) + accepted, reason = lanes.cli_lane_goal.judge_goal_completion( + "finish", + {"status": "incomplete", "summary": "still running"}, + open_request=lambda *_args, **_kwargs: pytest.fail("network must not run"), + ) + assert accepted is False + assert reason + + +def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs( + tmp_path: Path, +): + workspace = tmp_path / "workspace" + workspace.mkdir() + artifact = workspace / "evidence.txt" + artifact.write_text("evidence", encoding="utf-8") + outside = tmp_path / "outside.txt" + outside.write_text("outside", encoding="utf-8") + directory = workspace / "directory" + directory.mkdir() + + assert lanes.workspace_artifacts(workspace, "not-a-list") == [] + assert lanes.workspace_artifacts( + workspace, + [None, "", "missing", outside, directory, artifact, str(artifact)], + ) == [str(artifact.resolve())] + assert lanes._extract_json({"status": "completed"}) == {"status": "completed"} + assert lanes._extract_json(42) is None + assert lanes._extract_json("not json") is None + assert lanes._extract_json('prefix {"status":"blocked"} suffix') == { + "status": "blocked" + } + + +def test_provider_event_parsing_persists_sessions_and_nested_results( + tmp_path: Path, +): + state_file = tmp_path / "state.json" + state = {} + assert lanes._event_payload("codex", "not-json", state, state_file) is None + assert lanes._event_payload("codex", "[]", state, state_file) is None + assert ( + lanes._event_payload( + "codex", + json.dumps({"type": "thread.started", "thread": {"id": "thread-1"}}), + state, + state_file, + ) + is None + ) + assert state["codex_thread_id"] == "thread-1" + result = lanes._event_payload( + "claude", + json.dumps( + { + "session_id": "session-1", + "item": {"content": json.dumps(_completed_result("nested"))}, + } + ), + state, + state_file, + ) + assert result and result["summary"] == "nested" + assert state["claude_session_id"] == "session-1" + + +@pytest.mark.parametrize("assignee", ["cli-codex-max", "worker-auto", ""]) +def test_routing_rejects_invalid_assignees(assignee: str): + with pytest.raises(ValueError, match="unsupported"): + lanes.parse_assignee(assignee) + + +@pytest.mark.parametrize( + "target", + ["codex/model/high", "worker/local/model/high", "worker/codex/model/max"], +) +def test_worker_target_decoder_rejects_invalid_targets(target: str): + with pytest.raises(RuntimeError): + lanes._decode_worker_target(target) + + +def test_switchyard_network_failure_is_explicit(): + with pytest.raises(RuntimeError, match="routing failed"): + lanes.select_route( + "objective", + "cli-auto", + open_request=lambda *_args, **_kwargs: (_ for _ in ()).throw( + URLError("offline") + ), + ) diff --git a/testing/tests/test_hermes_cli_provider_edges.py b/testing/tests/test_hermes_cli_provider_edges.py new file mode 100644 index 00000000..21e57453 --- /dev/null +++ b/testing/tests/test_hermes_cli_provider_edges.py @@ -0,0 +1,173 @@ +"""Process lifecycle and provider artifact edge coverage for CLI workers.""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +from testing.tests.test_hermes_cli_support import _completed_result, lanes + + +def test_stream_process_enforces_runtime_and_lease_boundaries( + tmp_path: Path, +): + timed_out = lanes.stream_process( + [sys.executable, "-c", "import time; time.sleep(30)"], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "timeout.log", + state={}, + state_file=tmp_path / "timeout-state.json", + heartbeat=lambda _note: True, + max_runtime=-1, + ) + assert "maximum runtime" in timed_out.output + assert timed_out.returncode != 0 + + lease_lost = lanes.stream_process( + [sys.executable, "-c", "import time; time.sleep(30)"], + provider="claude", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "lease.log", + state={}, + state_file=tmp_path / "lease-state.json", + heartbeat=lambda _note: False, + max_runtime=60, + ) + assert "lease was lost" in lease_lost.output + assert lease_lost.returncode != 0 + + +def test_process_identity_and_signal_failures_are_bounded(monkeypatch): + assert lanes._process_record(999999999) is None + monkeypatch.setattr( + lanes, + "_process_identity_matches", + lambda _pid, _start: True, + ) + monkeypatch.setattr( + lanes.os, + "killpg", + lambda *_args: (_ for _ in ()).throw(ProcessLookupError()), + ) + monkeypatch.setattr( + lanes.os, + "kill", + lambda *_args: (_ for _ in ()).throw(ProcessLookupError()), + ) + lanes._signal_worker_tree(10, {11: (12, 13)}, signal.SIGTERM) + + +def test_descendant_snapshot_follows_multiple_generations(monkeypatch): + entries = [Path("/proc/self"), Path("/proc/10"), Path("/proc/11"), Path("/proc/12")] + monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) + records = { + 10: (1, 10, 100), + 11: (10, 11, 101), + 12: (11, 12, 102), + } + monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) + assert lanes._descendant_processes(10) == {11: (11, 101), 12: (12, 102)} + + +def test_terminate_escalates_after_term_timeout(monkeypatch): + class Process: + pid = 321 + waits = 0 + + @classmethod + def poll(cls): + return None + + @classmethod + def wait(cls, timeout): + cls.waits += 1 + if cls.waits == 1: + raise subprocess.TimeoutExpired("worker", timeout) + return 0 + + signals = [] + monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {}) + monkeypatch.setattr( + lanes, + "_signal_worker_tree", + lambda _pid, _descendants, sig: signals.append(sig), + ) + lanes._terminate_worker_process(Process()) + assert signals == [signal.SIGTERM, signal.SIGKILL] + assert Process.waits == 2 + + +def test_codex_result_collision_and_chmod_failure_preserve_result( + tmp_path: Path, + monkeypatch, +): + state = {"run_id": 7, "result_sequence": 0} + state_file = tmp_path / "task.json" + first = lanes._result_path(state_file, 7, 1) + first.write_text("prior", encoding="utf-8") + result_paths = [] + + def stream(command, **_kwargs): + result_path = Path(command[command.index("-o") + 1]) + result_path.write_text(json.dumps(_completed_result("new result")), encoding="utf-8") + result_paths.append(result_path) + return lanes.ProcessResult(0, "", None, False) + + monkeypatch.setattr(lanes, "stream_process", stream) + monkeypatch.setattr( + lanes.Path, + "chmod", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("readonly")), + ) + route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + result = lanes.run_provider( + route, + "work", + tmp_path, + state, + state_file, + tmp_path / "log", + lambda _note: True, + 60, + ) + assert result.structured["summary"] == "new result" + assert result_paths[0].name.endswith("provider-2.result.json") + + +def test_codex_missing_thread_skips_colliding_restart_result( + tmp_path: Path, + monkeypatch, +): + state = {"run_id": 8, "codex_thread_id": "missing"} + state_file = tmp_path / "task.json" + second = lanes._result_path(state_file, 8, 2) + second.write_text("collision", encoding="utf-8") + calls = [] + + def stream(command, **_kwargs): + calls.append(command) + if len(calls) == 1: + return lanes.ProcessResult(1, lanes.NO_CODEX_THREAD, None, False) + return lanes.ProcessResult(0, "", None, False) + + monkeypatch.setattr(lanes, "stream_process", stream) + route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + lanes.run_provider( + route, + "work", + tmp_path, + state, + state_file, + tmp_path / "log", + lambda _note: True, + 60, + ) + restart = Path(calls[1][calls[1].index("-o") + 1]) + assert restart.name.endswith("provider-3.result.json") diff --git a/testing/tests/test_hermes_cli_quarantine_edges.py b/testing/tests/test_hermes_cli_quarantine_edges.py new file mode 100644 index 00000000..eff30408 --- /dev/null +++ b/testing/tests/test_hermes_cli_quarantine_edges.py @@ -0,0 +1,185 @@ +"""Degraded quarantine-path coverage for terminal journal safety.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from testing.tests.test_hermes_cli_support import lanes + + +def test_quarantine_refuses_missing_or_unsafe_board_directory( + tmp_path: Path, + capsys, +): + path = tmp_path / "missing" / "journal" + assert lanes._quarantine_terminal(path, None, "invalid") == path + assert "unsafe board directory" in capsys.readouterr().err + + regular_parent = tmp_path / "regular" + regular_parent.write_text("not a directory", encoding="utf-8") + unsafe = regular_parent / "journal" + assert lanes._quarantine_terminal(unsafe, None, "invalid") == unsafe + + +def test_quarantine_accepts_a_lexically_outside_but_pinned_path( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "other-root") + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"invalid") + + destination = lanes._quarantine_terminal(path, None, "bad reason / outside") + + assert destination.parent == board / "quarantine" + assert not path.exists() + + +def test_quarantine_rejects_changed_pinned_board_identity( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"invalid") + snapshot = lanes._open_terminal_recovery_snapshot(path) + assert snapshot is not None + real_fstat = lanes.os.fstat + + def changed_board(descriptor): + observed = real_fstat(descriptor) + if descriptor == snapshot.directory_descriptor: + return os.stat_result( + (*observed[:1], observed.st_ino + 1, *observed[2:]) + ) + return observed + + monkeypatch.setattr(lanes.os, "fstat", changed_board) + try: + lanes._quarantine_terminal(path, None, "changed-board", snapshot=snapshot) + finally: + snapshot.close() + assert not path.exists() + assert "degraded safely" in capsys.readouterr().err + + +def test_quarantine_rejects_source_replaced_during_open( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"original") + replacement = board / "replacement" + replacement.write_bytes(b"replacement") + real_open = lanes.os.open + swapped = {"value": False} + + def swap_before_source_open(target, flags, *args, **kwargs): + if target == path.name and kwargs.get("dir_fd") is not None and not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + return real_open(target, flags, *args, **kwargs) + + monkeypatch.setattr(lanes.os, "open", swap_before_source_open) + lanes._quarantine_terminal(path, None, "source-race") + assert path.exists() + assert path.read_bytes() == b"replacement" + assert "degraded safely" in capsys.readouterr().err + + +def test_quarantine_unlinks_an_invalid_destination_before_degrading( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"invalid") + real_fstat = lanes.os.fstat + target_descriptors = set() + real_fdopen = lanes.os.fdopen + + def remember_target(descriptor, *args, **kwargs): + target_descriptors.add(descriptor) + return real_fdopen(descriptor, *args, **kwargs) + + def unsafe_target(descriptor): + observed = real_fstat(descriptor) + if descriptor in target_descriptors: + return os.stat_result( + (*observed[:3], 0o100644, *observed[4:]) + ) + return observed + + monkeypatch.setattr(lanes.os, "fdopen", remember_target) + monkeypatch.setattr(lanes.os, "fstat", unsafe_target) + lanes._quarantine_terminal(path, None, "unsafe-target") + + assert not list((board / "quarantine").glob("*.quarantine")) + assert "degraded safely" in capsys.readouterr().err + + +def test_quarantine_destination_collision_is_bounded( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"invalid") + real_open = lanes.os.open + + def collide_on_destination(target, flags, *args, **kwargs): + if ( + isinstance(target, str) + and target.endswith(".quarantine") + and flags & os.O_EXCL + ): + raise FileExistsError(target) + return real_open(target, flags, *args, **kwargs) + + monkeypatch.setattr(lanes.os, "open", collide_on_destination) + destination = lanes._quarantine_terminal(path, None, "collision") + + assert destination == path + assert not path.exists() + assert "could not reserve" in capsys.readouterr().err + + +def test_quarantine_deferred_retirement_does_not_raise( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + board = tmp_path / "board" + board.mkdir() + path = board / "journal" + path.write_bytes(b"invalid") + calls = {"value": 0} + + def fail_retirement(*_args, **_kwargs): + calls["value"] += 1 + raise OSError("retirement unavailable") + + monkeypatch.setattr(lanes, "_retire_terminal_entry", fail_retirement) + destination = lanes._quarantine_terminal(path, None, "retire-failure") + + assert destination.exists() + assert path.exists() + assert calls["value"] == 2 + assert "retirement=deferred" in capsys.readouterr().err diff --git a/testing/tests/test_hermes_cli_records_edges.py b/testing/tests/test_hermes_cli_records_edges.py new file mode 100644 index 00000000..3362466a --- /dev/null +++ b/testing/tests/test_hermes_cli_records_edges.py @@ -0,0 +1,230 @@ +"""Bounded-read and terminal-schema edge coverage for CLI lane records.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_cli_support import ( + _completed_result, + _pending_terminal_record, + lanes, +) + + +def test_bounded_reader_stops_at_eof_and_zero_limit(tmp_path: Path): + path = tmp_path / "payload" + path.write_bytes(b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + assert lanes._read_bounded(descriptor, 10) == b"abc" + os.lseek(descriptor, 0, os.SEEK_SET) + assert lanes._read_bounded(descriptor, 0) == b"" + finally: + os.close(descriptor) + + +def test_small_snapshot_rejects_nonregular_hardlinked_oversized_and_nonobject( + tmp_path: Path, + monkeypatch, +): + directory = tmp_path / "directory" + directory.mkdir() + assert lanes._open_small_json_snapshot(directory) is None + + original = tmp_path / "original" + original.write_text("{}", encoding="utf-8") + hardlink = tmp_path / "hardlink" + os.link(original, hardlink) + assert lanes._open_small_json_snapshot(hardlink) is None + + oversized = tmp_path / "oversized" + oversized.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 1) + assert lanes._open_small_json_snapshot(oversized) is None + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 1024 * 1024) + + array = tmp_path / "array" + array.write_text("[]", encoding="utf-8") + assert lanes._open_small_json_snapshot(array) is None + malformed = tmp_path / "malformed" + malformed.write_bytes(b"\xff") + assert lanes._open_small_json_snapshot(malformed) is None + + +def test_small_snapshot_rejects_path_replaced_after_read( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "record" + path.write_text("{}", encoding="utf-8") + real_stat = lanes.os.stat + calls = {"value": 0} + + def changed_stat(target, *args, **kwargs): + result = real_stat(target, *args, **kwargs) + if target == path.name and kwargs.get("dir_fd") is not None: + calls["value"] += 1 + if calls["value"] == 1: + return os.stat_result( + (*result[:6], result.st_size + 1, *result[7:]) + ) + return result + + monkeypatch.setattr(lanes.os, "stat", changed_stat) + assert lanes._open_small_json_snapshot(path) is None + + +def test_recovery_snapshot_classifies_nonregular_and_open_failure( + tmp_path: Path, + monkeypatch, +): + fifo = tmp_path / "fifo" + os.mkfifo(fifo) + snapshot = lanes._open_terminal_recovery_snapshot(fifo) + assert snapshot is not None + assert snapshot.invalid_reason == "non-regular" + snapshot.close() + + path = tmp_path / "record" + path.write_text("{}", encoding="utf-8") + real_open = lanes.os.open + + def fail_file_open(target, flags, *args, **kwargs): + if target == path.name: + raise OSError("denied") + return real_open(target, flags, *args, **kwargs) + + monkeypatch.setattr(lanes.os, "open", fail_file_open) + snapshot = lanes._open_terminal_recovery_snapshot(path) + assert snapshot is not None + assert snapshot.invalid_reason == "open-failed" + snapshot.close() + + +def test_recovery_snapshot_detects_identity_change_during_open( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "record" + path.write_text("{}", encoding="utf-8") + replacement = tmp_path / "replacement" + replacement.write_text('{"replacement":true}', encoding="utf-8") + real_open = lanes.os.open + swapped = {"value": False} + + def swap_before_file_open(target, flags, *args, **kwargs): + if target == path.name and not swapped["value"]: + os.replace(replacement, path) + swapped["value"] = True + return real_open(target, flags, *args, **kwargs) + + monkeypatch.setattr(lanes.os, "open", swap_before_file_open) + snapshot = lanes._open_terminal_recovery_snapshot(path) + assert snapshot is not None + assert snapshot.invalid_reason == "identity-changed-during-open" + snapshot.close() + + +def test_recovery_snapshot_bounds_hardlinks_oversize_and_bad_payloads( + tmp_path: Path, + monkeypatch, +): + original = tmp_path / "original" + original.write_text("{}", encoding="utf-8") + hardlink = tmp_path / "hardlink" + os.link(original, hardlink) + snapshot = lanes._open_terminal_recovery_snapshot(hardlink) + assert snapshot is not None and snapshot.invalid_reason == "hardlinked" + snapshot.close() + + oversized = tmp_path / "oversized" + oversized.write_bytes(b"x" * 128) + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 64) + monkeypatch.setattr(lanes, "QUARANTINE_HASH_BYTES", 8) + snapshot = lanes._open_terminal_recovery_snapshot(oversized) + assert snapshot is not None and snapshot.invalid_reason == "oversized" + assert snapshot.prefix == b"x" * 8 + snapshot.close() + + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 1024 * 1024) + array = tmp_path / "array" + array.write_text("[]", encoding="utf-8") + snapshot = lanes._open_terminal_recovery_snapshot(array) + assert snapshot is not None and snapshot.invalid_reason == "non-object-payload" + snapshot.close() + malformed = tmp_path / "malformed" + malformed.write_bytes(b"\xff") + snapshot = lanes._open_terminal_recovery_snapshot(malformed) + assert snapshot is not None and snapshot.invalid_reason == "malformed-payload" + snapshot.close() + + +def test_snapshot_loaders_return_empty_for_missing_or_wrong_identity( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + identity = lanes.TerminalIdentity("board", "task", 1, "pending") + missing = lanes.STATE_ROOT / "board" / "task.run-1.terminal.pending.json" + assert lanes._open_terminal_snapshot(missing, identity) is None + assert lanes._load_terminal_json(missing, identity) == {} + assert lanes._load_small_json(missing) == {} + + +def test_candidate_persistence_skips_existing_sequence(tmp_path: Path): + state_file = tmp_path / "task.json" + state = {"board": "board", "task_id": "task", "run_id": 4} + route = lanes.Route("codex", "gpt", "high", "profile", "test", "test", 1, ()) + first = lanes._candidate_path(state_file, 4, 1) + first.write_text("collision", encoding="utf-8") + path = lanes._persist_candidate( + state, + state_file, + _completed_result(), + route=route, + returncode=0, + goal_turn=2, + ) + assert path.name.endswith("candidate-2.json") + assert state["candidate_sequence"] == 2 + + +@pytest.mark.parametrize( + "mutation", + [ + lambda record: [], + lambda record: {**record, "board": ""}, + lambda record: {**record, "expected_run_id": True}, + lambda record: {**record, "metadata": []}, + lambda record: {**record, "result": "not-json"}, + lambda record: {**record, "result": "[]"}, + lambda record: { + **record, + "result": json.dumps({**_completed_result(), "extra": True}), + }, + lambda record: { + **record, + "result": json.dumps({**_completed_result(), "summary": ""}), + }, + lambda record: { + **record, + "result": json.dumps({**_completed_result(), "tests_run": [1]}), + }, + ], +) +def test_terminal_schema_rejects_each_malformed_boundary(mutation): + record = _pending_terminal_record("board", "task", 1, "done") + assert lanes._terminal_record_valid(mutation(record)) is False + + +def test_terminal_schema_validates_optional_identity(): + record = _pending_terminal_record("board", "task", 1, "done") + assert lanes._terminal_record_valid(record) is True + assert lanes._terminal_record_valid( + record, + lanes.TerminalIdentity("other", "task", 1, "pending"), + ) is False diff --git a/testing/tests/test_hermes_cli_recovery_edges.py b/testing/tests/test_hermes_cli_recovery_edges.py new file mode 100644 index 00000000..27ba6773 --- /dev/null +++ b/testing/tests/test_hermes_cli_recovery_edges.py @@ -0,0 +1,298 @@ +"""Recovery convergence and hidden-authority edge coverage.""" + +from __future__ import annotations + +import hashlib +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_support import ( + _pending_terminal_record, + lanes, +) + + +class _Snapshot: + def __init__(self, document=None): + self.document = document + self.closed = 0 + + def close(self): + self.closed += 1 + + +def test_terminal_absence_distinguishes_present_missing_and_inaccessible( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "journal" + path.write_text("data", encoding="utf-8") + assert lanes._terminal_entry_absent(path) is False + path.unlink() + assert lanes._terminal_entry_absent(path) is True + monkeypatch.setattr( + lanes.os, + "open", + lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("denied")), + ) + assert lanes._terminal_entry_absent(path) is False + + +def _staged_path(root: Path, identity, document: dict) -> tuple[Path, Path]: + if identity.state in {"pending", "committed"}: + canonical = lanes._terminal_path( + lanes.state_path(identity.board, identity.task_id), + identity.run_id, + identity.state, + ) + else: + canonical = lanes._terminal_evidence_path(identity, identity.state, document) + digest = hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16] + return root / identity.board / f".retire.{digest}.{'a' * 16}.0", canonical + + +@pytest.mark.parametrize("state", ["prepared", "conflict"]) +def test_staged_authority_accepts_valid_evidence_states( + tmp_path: Path, + monkeypatch, + state: str, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + identity = lanes.TerminalIdentity("board", "task", 5, state) + document = _pending_terminal_record("board", "task", 5, "accepted") + document["kanban_state"] = state + staged, canonical = _staged_path(lanes.STATE_ROOT, identity, document) + staged.parent.mkdir(parents=True) + authority = lanes._staged_terminal_authority(staged, _Snapshot(document)) + assert authority == (identity, canonical) + + +def test_staged_authority_rejects_outside_nested_and_unknown_state( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + document = _pending_terminal_record("board", "task", 5, "accepted") + outside = tmp_path / ".retire.x" + assert lanes._staged_terminal_authority(outside, _Snapshot(document)) is None + nested = lanes.STATE_ROOT / "board" / "nested" / ".retire.name" + assert lanes._staged_terminal_authority(nested, _Snapshot(document)) is None + board = lanes.STATE_ROOT / "board" + board.mkdir(parents=True) + staged = board / f".retire.{'a' * 16}.{'b' * 16}.0" + document["kanban_state"] = "unknown" + assert lanes._staged_terminal_authority(staged, _Snapshot(document)) is None + + +def test_restore_staged_authority_handles_every_first_writer_state( + tmp_path: Path, + monkeypatch, +): + document = _pending_terminal_record("board", "task", 6, "accepted") + canonical = tmp_path / "canonical" + pending = lanes.TerminalIdentity("board", "task", 6, "pending") + calls = [] + monkeypatch.setattr( + lanes, + "_persist_prepared_evidence", + lambda identity, value: calls.append(("prepared", identity, value)), + ) + lanes._restore_staged_terminal_authority(pending, canonical, document) + assert calls[0][0] == "prepared" + + monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: True) + committed = lanes.TerminalIdentity("board", "task", 6, "committed") + lanes._restore_staged_terminal_authority(committed, canonical, document) + + monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False) + monkeypatch.setattr(lanes, "_load_small_json", lambda _path: {}) + monkeypatch.setattr( + lanes, + "_persist_conflict_evidence", + lambda *args: calls.append(("conflict", *args)), + ) + lanes._restore_staged_terminal_authority(committed, canonical, document) + assert calls[-1][0] == "conflict" + + prepared = lanes.TerminalIdentity("board", "task", 6, "prepared") + monkeypatch.setattr(lanes, "_terminal_evidence_valid", lambda *_args: False) + with pytest.raises(OSError, match="conflicting result"): + lanes._restore_staged_terminal_authority(prepared, canonical, document) + + +def test_restore_staged_authority_accepts_identical_existing_evidence( + tmp_path: Path, + monkeypatch, +): + document = _pending_terminal_record("board", "task", 7, "accepted") + document["kanban_state"] = "prepared" + identity = lanes.TerminalIdentity("board", "task", 7, "prepared") + monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False) + monkeypatch.setattr(lanes, "_load_small_json", lambda _path: document) + monkeypatch.setattr(lanes, "_terminal_evidence_valid", lambda *_args: True) + lanes._restore_staged_terminal_authority(identity, tmp_path / "existing", document) + + +def test_staging_recovery_bounds_missing_snapshots_and_restore_errors( + monkeypatch, +): + path = Path("/tmp/staged") + monkeypatch.setattr(lanes, "_retirement_staging_paths", lambda: [path]) + monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: None) + assert lanes._recover_retirement_staging() == 0 + + identity = lanes.TerminalIdentity("board", "task", 8, "pending") + snapshot = _Snapshot(_pending_terminal_record("board", "task", 8, "accepted")) + monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: snapshot) + monkeypatch.setattr( + lanes, + "_staged_terminal_authority", + lambda *_args: (identity, Path("/tmp/canonical")), + ) + monkeypatch.setattr( + lanes, + "_restore_staged_terminal_authority", + lambda *_args: (_ for _ in ()).throw(OSError("restore")), + ) + errors = [] + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: errors.append((board, str(error))), + ) + assert lanes._recover_retirement_staging() == 0 + assert errors == [("board", "restore")] + assert snapshot.closed == 1 + + +def test_drain_staging_ignores_disappearing_entries(monkeypatch): + class Vanished: + def stat(self, **_kwargs): + raise FileNotFoundError + + monkeypatch.setattr(lanes, "_retirement_staging_paths", lambda: [Vanished()]) + assert lanes._drain_retirement_staging() == 0 + + +def _prepared_path(tmp_path: Path, monkeypatch) -> tuple[Path, dict]: + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + pending = _pending_terminal_record("board", "task", 9, "accepted") + identity = lanes.TerminalIdentity("board", "task", 9, "pending") + path = lanes._persist_prepared_evidence(identity, pending) + return path, pending + + +def test_prepared_recovery_defers_without_discarding_evidence( + tmp_path: Path, + monkeypatch, +): + path, _document = _prepared_path(tmp_path, monkeypatch) + monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "deferred") + assert lanes._recover_prepared_finalizations(object()) == 0 + assert path.exists() + + +def test_prepared_recovery_bounds_db_and_publication_exceptions( + tmp_path: Path, + monkeypatch, +): + path, _document = _prepared_path(tmp_path, monkeypatch) + errors = [] + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: errors.append((board, str(error))), + ) + monkeypatch.setattr( + lanes, + "_finalize_document_db", + lambda *_args: (_ for _ in ()).throw(OSError("database")), + ) + assert lanes._recover_prepared_finalizations(object()) == 0 + assert errors[-1] == ("board", "database") + assert path.exists() + + monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "committed") + monkeypatch.setattr( + lanes, + "_promote_prepared_evidence", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("publish")), + ) + assert lanes._recover_prepared_finalizations(object()) == 0 + assert errors[-1] == ("board", "publish") + + +def test_pending_recovery_bounds_finalizer_errors_and_replacement_churn( + tmp_path: Path, + monkeypatch, + capsys, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + path = lanes._terminal_path(lanes.state_path("board", "task"), 10) + lanes.atomic_json(path, _pending_terminal_record("board", "task", 10, "accepted")) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=object())) + errors = [] + monkeypatch.setattr( + lanes, + "_record_board_access_error", + lambda board, error: errors.append((board, str(error))), + ) + monkeypatch.setattr( + lanes, + "_finalize_terminal_record", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("finalize")), + ) + assert lanes.recover_pending_finalizations() == 0 + assert errors[-1] == ("board", "finalize") + + def invalid(*_args, snapshot=None, **_kwargs): + snapshot.close() + return "invalid" + + monkeypatch.setattr(lanes, "_finalize_terminal_record", invalid) + assert lanes.recover_pending_finalizations() == 0 + assert "replacement churn deferred" in capsys.readouterr().err + + +def test_pending_recovery_classifies_foreign_identity( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + path = lanes._terminal_path(lanes.state_path("board", "task"), 11) + lanes.atomic_json(path, _pending_terminal_record("other", "task", 11, "accepted")) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=object())) + reasons = [] + + def quarantine(_path, _identity, reason, *, snapshot): + reasons.append(reason) + os.unlink(_path) + + monkeypatch.setattr(lanes, "_quarantine_terminal", quarantine) + monkeypatch.setattr(lanes, "_recover_exact_run", lambda *_args: False) + assert lanes.recover_pending_finalizations() == 0 + assert reasons == ["foreign-identity"] + + +def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + assert lanes._has_pending_finalization("board", "task", True) is False + path = lanes._terminal_path(lanes.state_path("board", "task"), 12) + path.parent.mkdir(parents=True) + path.write_text("bad", encoding="utf-8") + real_stat = Path.stat + + def denied(candidate, *args, **kwargs): + if candidate == path: + raise PermissionError("denied") + return real_stat(candidate, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", denied) + assert lanes._has_pending_finalization("board", "task", 12) is False diff --git a/testing/tests/test_hermes_cli_retention_edges.py b/testing/tests/test_hermes_cli_retention_edges.py new file mode 100644 index 00000000..f7deb6a2 --- /dev/null +++ b/testing/tests/test_hermes_cli_retention_edges.py @@ -0,0 +1,89 @@ +"""Retained terminal classification and filesystem-failure edge coverage.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.tests.test_hermes_cli_support import ( + _pending_terminal_record, + lanes, +) + + +class _Snapshot: + document = None + invalid_reason = "malformed" + + def __init__(self): + self.closed = 0 + + def close(self): + self.closed += 1 + + +def test_retained_classifier_handles_missing_snapshot(tmp_path: Path, monkeypatch): + path = tmp_path / "task.run-1.terminal.committed.json" + monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: None) + assert lanes._quarantine_invalid_retained_terminal(path) == (True, False) + + +def test_retained_classifier_accepts_valid_committed_evidence( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + state_file = lanes.state_path("board", "task") + path = lanes._terminal_path(state_file, 2, "committed") + record = _pending_terminal_record("board", "task", 2, "accepted") + record["kanban_state"] = "committed" + lanes.atomic_json(path, record) + assert lanes._quarantine_invalid_retained_terminal(path) == (False, False) + + +def test_retained_classifier_bounds_post_quarantine_stat_error( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "task.run-3.terminal.committed.json" + snapshot = _Snapshot() + monkeypatch.setattr( + lanes, + "_open_terminal_recovery_snapshot", + lambda _path: snapshot, + ) + monkeypatch.setattr(lanes, "_quarantine_terminal", lambda *_args, **_kwargs: path) + real_stat = Path.stat + + def fail_selected(candidate, *args, **kwargs): + if candidate == path: + raise PermissionError("denied") + return real_stat(candidate, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", fail_selected) + assert lanes._quarantine_invalid_retained_terminal(path) == (True, False) + assert snapshot.closed == 1 + + +def test_gc_skips_symlink_board_and_disappearing_candidate( + tmp_path: Path, + monkeypatch, +): + root = tmp_path / "lanes" + root.mkdir() + target = tmp_path / "target" + target.mkdir() + (root / "linked-board").symlink_to(target, target_is_directory=True) + board = root / "board" + board.mkdir() + candidate = board / "task.candidate-1.json" + candidate.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "STATE_ROOT", root) + real_stat = Path.stat + + def disappear(selected, *args, **kwargs): + if selected == candidate: + raise FileNotFoundError + return real_stat(selected, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", disappear) + assert lanes.gc_lane_artifacts() == 0 From e8c26ecf85a0c802d378fffffb98b7df43b78242 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:04:17 -0300 Subject: [PATCH 13/46] hermes: add daemonless agent image release lane --- ci/Jenkinsfile.hermes-agent-image | 288 ++++++++++ ci/scripts/hermes_image_release.py | 500 ++++++++++++++++++ .../applications/harbor/kustomization.yaml | 8 +- .../applications/hermes/kustomization.yaml | 1 + .../applications/jenkins/kustomization.yaml | 5 +- .../applications/kustomization.yaml | 1 + .../kustomization.yaml | 26 + .../applications/vault/kustomization.yaml | 5 + dockerfiles/Dockerfile.hermes-agent | 15 + .../Dockerfile.hermes-agent.dockerignore | 2 + dockerfiles/hermes-kaniko-heredoc-runner.py | 146 +++++ .../harbor/hermes-agent-immutability-job.yaml | 79 +++ services/harbor/kustomization.yaml | 5 + .../policy-bootstrap-serviceaccount.yaml | 7 + ...harbor_hermes_agent_immutability_ensure.py | 378 +++++++++++++ services/hermes/NOTES.md | 104 ++++ services/hermes/agent-deployment.yaml | 5 + services/hermes/kustomization.yaml | 1 + .../scripts/jenkins_image_build_trigger.py | 112 ++++ .../hermes/scripts/stage_runtime_access.py | 1 + services/hermes/switchyard-deployment.yaml | 2 +- services/jenkins/configmap-jcasc.yaml | 19 + services/jenkins/configmap-plugins.yaml | 1 + services/jenkins/deployment.yaml | 3 + .../hermes-image-builder-serviceaccount.yaml | 7 + services/jenkins/kustomization.yaml | 1 + services/jenkins/secretproviderclass.yaml | 2 +- .../vault-hermes-jenkins-token-seed/job.yaml | 68 +++ .../kustomization.yaml | 13 + ...vault_hermes_jenkins_build_token_ensure.sh | 167 ++++++ .../serviceaccount.yaml | 7 + .../vault/hermes-auth-role-bootstrap-job.yaml | 30 +- .../vault/scripts/vault_k8s_auth_configure.sh | 37 +- testing/quality_contract.json | 15 + testing/tests/test_hermes_image_builder.py | 473 +++++++++++++++++ .../test_hermes_image_builder_adversarial.py | 469 ++++++++++++++++ .../test_hermes_image_builder_coverage.py | 456 ++++++++++++++++ .../test_hermes_image_builder_fresh_review.py | 335 ++++++++++++ .../tests/test_hermes_image_builder_harbor.py | 325 ++++++++++++ .../tests/test_hermes_image_builder_vault.py | 329 ++++++++++++ testing/tests/test_hermes_runtime_access.py | 2 + 41 files changed, 4439 insertions(+), 11 deletions(-) create mode 100644 ci/Jenkinsfile.hermes-agent-image create mode 100755 ci/scripts/hermes_image_release.py create mode 100644 clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml create mode 100644 dockerfiles/hermes-kaniko-heredoc-runner.py create mode 100644 services/harbor/hermes-agent-immutability-job.yaml create mode 100644 services/harbor/policy-bootstrap-serviceaccount.yaml create mode 100644 services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py create mode 100755 services/hermes/scripts/jenkins_image_build_trigger.py create mode 100644 services/jenkins/hermes-image-builder-serviceaccount.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/job.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/kustomization.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh create mode 100644 services/vault-hermes-jenkins-token-seed/serviceaccount.yaml create mode 100644 testing/tests/test_hermes_image_builder.py create mode 100644 testing/tests/test_hermes_image_builder_adversarial.py create mode 100644 testing/tests/test_hermes_image_builder_coverage.py create mode 100644 testing/tests/test_hermes_image_builder_fresh_review.py create mode 100644 testing/tests/test_hermes_image_builder_harbor.py create mode 100644 testing/tests/test_hermes_image_builder_vault.py diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image new file mode 100644 index 00000000..2d350ac7 --- /dev/null +++ b/ci/Jenkinsfile.hermes-agent-image @@ -0,0 +1,288 @@ +pipeline { + agent { + kubernetes { + defaultContainer 'python' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-04 + - titan-14 + - titan-18 + - titan-19 + - titan-22 + - titan-24 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + - name: python + image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808 + command: ["sleep"] + args: ["99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi +""" + } + } + parameters { + booleanParam( + name: 'PUBLISH_IMAGE', + defaultValue: false, + description: 'Publish the reviewed main revision to Harbor.' + ) + string( + name: 'EXPECTED_SOURCE_REVISION', + defaultValue: '', + description: 'Exact 40-character commit on atlas/titan-iac main.' + ) + string( + name: 'CONFIRM_PUBLISH', + defaultValue: '', + description: 'Enter PUBLISH HERMES AGENT to confirm the release.' + ) + } + environment { + HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent' + } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100')) + skipDefaultCheckout(true) + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout reviewed source') { + steps { + checkout scm + } + } + stage('Enforce release boundary') { + steps { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test "${actual_revision}" = "$(git rev-parse origin/main)" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + ''' + } + } + stage('Reject replay before publish') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py assert-absent \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" + ''' + } + } + } + stage('Build and publish without a daemon') { + steps { + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + /kaniko/executor \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ + --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --cleanup \ + --push-retry=3 + ''' + } + } + } + } + stage('Render reviewed Flux handoff') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py render \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + test -s build/hermes-agent-release/hermes-image-update.patch + test -s build/hermes-agent-release/hermes-agent-image.json + ''' + } + } + } + } + post { + success { + sh ''' + set -eu + expected_files="$(printf '%s\n' \ + build/hermes-agent.destination \ + build/hermes-agent.digest \ + build/hermes-agent.image \ + build/hermes-agent-release/hermes-agent-image.json \ + build/hermes-agent-release/hermes-image-update.patch \ + build/hermes-agent-release/hermes-kustomization.yaml \ + | LC_ALL=C sort)" + actual_files="$(find build -type f -print | LC_ALL=C sort)" + test "${actual_files}" = "${expected_files}" + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py verify-evidence \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + ''' + archiveArtifacts( + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + allowEmptyArchive: false, + fingerprint: true + ) + } + cleanup { + container('kaniko') { + sh '''#!/busybox/sh + /busybox/rm -f /kaniko/.docker/config.json + ''' + } + } + } +} diff --git a/ci/scripts/hermes_image_release.py b/ci/scripts/hermes_image_release.py new file mode 100755 index 00000000..2f0f4166 --- /dev/null +++ b/ci/scripts/hermes_image_release.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Verify and render a reviewable Hermes agent image release.""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent" +HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0" +HARBOR_PROJECT = "bstein" +HARBOR_REPOSITORY = "hermes-agent" +IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent" +IMMUTABLE_TAG_PATTERN = "git-*-build-*" +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$") +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$" +) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never send registry credentials to a redirect target.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _validated(value: str, pattern: re.Pattern[str], label: str) -> str: + """Return a normalized value when it matches the release contract.""" + normalized = value.strip() + if not pattern.fullmatch(normalized): + raise ValueError(f"invalid {label}: expected {pattern.pattern}") + return normalized + + +def validate_destination( + destination: str, source_revision: str, build_number: str +) -> tuple[str, str]: + """Bind one unique build tag to the reviewed revision and Jenkins build.""" + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + build = _validated(build_number, BUILD_PATTERN, "build number") + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match or match.groups() != (revision, build): + raise ValueError( + "destination must bind the reviewed revision and unique Jenkins build" + ) + return revision, build + + +def validate_kaniko_evidence( + *, digest_text: str, image_text: str, destination: str +) -> str: + """Cross-check both independent Kaniko output files against the destination.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest") + if image_lines[0].strip() != f"{destination}@{digest}": + raise ValueError("Kaniko image evidence does not match destination and digest") + return digest + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Make a registry request without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _artifact_response( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes]: + """Read one exact Harbor artifact by tag with bounded response size.""" + match = DESTINATION_PATTERN.fullmatch(destination) + if not match: + raise ValueError("invalid destination") + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + tag = destination.rsplit(":", 1)[1] + encoded_tag = urllib.parse.quote(tag, safe="") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/" + f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}" + "?with_immutable_status=true", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor artifact response exceeded the size limit") + return int(response.status), body + + +def _immutable_rules_response( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes, dict[str, str]]: + """Read the project policy with the same least-privilege publish identity.""" + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules" + "?page=1&page_size=100", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor immutable rule response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def _require_complete_rule_page( + rules: list[dict[str, Any]], headers: dict[str, str] +) -> None: + """Require proof that the bounded first page contains every rule.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError("Harbor immutable rule list omitted a valid total count") + if int(raw_total) != len(rules): + raise RuntimeError("Harbor immutable rule list was truncated") + + +def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Select only fields that bind the server-side build-tag policy.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in rule.get("tag_selectors") or [] + if isinstance(item, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(item, dict) + ] + }, + } + + +def verify_immutable_policy( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Fail closed before build unless the exact Harbor rule is active.""" + status, body, headers = _immutable_rules_response( + username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}") + try: + rules = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc + if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules): + raise RuntimeError("Harbor immutable rule list has an invalid shape") + _require_complete_rule_page(rules, headers) + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": IMMUTABLE_TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": IMMUTABLE_REPOSITORY_PATTERN, + } + ] + }, + } + matches = [ + _normalized_immutable_rule(item) + for item in rules + if _normalized_immutable_rule(item)["tag_selectors"] + == expected["tag_selectors"] + and _normalized_immutable_rule(item)["scope_selectors"] + == expected["scope_selectors"] + ] + if matches != [expected]: + raise RuntimeError("Harbor immutable build-tag policy is absent or not exact") + + +def assert_tag_absent( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Reject replay before Kaniko can push an already-used immutable identity.""" + status, _body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status == 404: + return + if status == 200: + raise RuntimeError("Harbor destination tag already exists; refusing overwrite") + raise RuntimeError(f"Harbor destination preflight returned HTTP {status}") + + +def verify_registry_digest( + destination: str, + digest: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Verify Harbor independently resolves the pushed tag to Kaniko's digest.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + status, body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor manifest verification returned HTTP {status}") + try: + artifact = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid artifact JSON") from exc + harbor_digest = str(artifact.get("digest") or "").strip() + if not DIGEST_PATTERN.fullmatch(harbor_digest): + raise RuntimeError("Harbor response omitted a valid artifact digest") + if harbor_digest != digest: + raise RuntimeError("Harbor digest does not match Kaniko evidence") + expected_tag = destination.rsplit(":", 1)[1] + matching_tags = [ + item + for item in artifact.get("tags") or [] + if isinstance(item, dict) and item.get("name") == expected_tag + ] + if len(matching_tags) != 1: + raise RuntimeError("Harbor artifact does not contain the expected tag") + if matching_tags[0].get("immutable") is not True: + raise RuntimeError("Harbor did not enforce the expected tag as immutable") + + +def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str: + """Replace exactly one matching Kustomize image digest without reformatting.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + lines = source.splitlines(keepends=True) + matches: list[int] = [] + + for index, line in enumerate(lines): + if line.strip() != f"- name: {image}": + continue + name_indent = len(line) - len(line.lstrip()) + for candidate_index in range(index + 1, len(lines)): + candidate = lines[candidate_index] + stripped = candidate.strip() + candidate_indent = len(candidate) - len(candidate.lstrip()) + if stripped.startswith("- name:") and candidate_indent == name_indent: + break + if stripped.startswith("digest:") and candidate_indent > name_indent: + matches.append(candidate_index) + break + + if len(matches) != 1: + raise ValueError( + f"expected exactly one digest for image {image!r}; found {len(matches)}" + ) + + index = matches[0] + newline = "\n" if lines[index].endswith("\n") else "" + prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] + lines[index] = f"{prefix}digest: {digest}{newline}" + return "".join(lines) + + +def write_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> dict[str, str]: + """Write a rendered manifest, patch, and credential-free release metadata.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + if not patch: + raise ValueError("published digest already matches the Flux manifest") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8") + (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8") + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + (output_dir / "hermes-agent-image.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return metadata + + +def validate_release_artifacts( + *, + digest_file: Path, + image_file: Path, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> None: + """Revalidate the exact successful-build evidence without rewriting it.""" + digest = validate_kaniko_evidence( + digest_text=digest_file.read_text(encoding="utf-8"), + image_text=image_file.read_text(encoding="utf-8"), + destination=destination, + ) + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + expected_names = { + "hermes-agent-image.json", + "hermes-image-update.patch", + "hermes-kustomization.yaml", + } + entries = list(output_dir.iterdir()) + if {entry.name for entry in entries} != expected_names or not all( + entry.is_file() and not entry.is_symlink() for entry in entries + ): + raise ValueError("release output must contain exactly three evidence files") + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + expected = { + "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n", + "hermes-image-update.patch": patch, + "hermes-kustomization.yaml": rendered, + } + for name, expected_text in expected.items(): + if (output_dir / name).read_text(encoding="utf-8") != expected_text: + raise ValueError(f"release evidence is incomplete or mismatched: {name}") + + +def _credentials() -> tuple[str, str]: + """Read the masked, runtime-only Jenkins credential environment.""" + username = os.environ.get("HARBOR_USER", "") + password = os.environ.get("HARBOR_PASSWORD", "") + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + return username, password + + +def _common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--destination", required=True) + + +def main() -> int: + """Fail closed around the unique tag, then verify and render after push.""" + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + absent = commands.add_parser("assert-absent") + _common_arguments(absent) + render = commands.add_parser("render") + _common_arguments(render) + render.add_argument("--digest-file", required=True, type=Path) + render.add_argument("--image-file", required=True, type=Path) + render.add_argument("--kustomization", required=True, type=Path) + render.add_argument("--output-dir", required=True, type=Path) + verify = commands.add_parser("verify-evidence") + _common_arguments(verify) + verify.add_argument("--digest-file", required=True, type=Path) + verify.add_argument("--image-file", required=True, type=Path) + verify.add_argument("--kustomization", required=True, type=Path) + verify.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + + validate_destination(args.destination, args.source_revision, args.build_number) + if args.command == "verify-evidence": + validate_release_artifacts( + digest_file=args.digest_file, + image_file=args.image_file, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + username, password = _credentials() + if args.command == "assert-absent": + verify_immutable_policy(username=username, password=password) + assert_tag_absent(args.destination, username=username, password=password) + return 0 + + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + verify_registry_digest( + args.destination, digest, username=username, password=password + ) + write_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml index 5bbe2e8f..225214fe 100644 --- a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml @@ -15,7 +15,13 @@ spec: kind: GitRepository name: flux-system namespace: flux-system - wait: false + wait: true + timeout: 10m + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor dependsOn: - name: core - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..9a8365ea 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,4 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: jenkins diff --git a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml index dce79ab7..5cf8d56d 100644 --- a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml @@ -6,10 +6,9 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent - atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance." spec: interval: 10m - suspend: true + suspend: false path: ./services/jenkins prune: true sourceRef: @@ -18,6 +17,8 @@ spec: targetNamespace: jenkins dependsOn: - name: helm + - name: harbor + - name: vault-hermes-jenkins-token-seed healthChecks: - apiVersion: apps/v1 kind: Deployment diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..3303f4fa 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - gitea/kustomization.yaml - vault/kustomization.yaml + - vault-hermes-jenkins-token-seed/kustomization.yaml - vaultwarden/kustomization.yaml - comms/kustomization.yaml - crypto/kustomization.yaml diff --git a/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..561179c8 --- /dev/null +++ b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,26 @@ +# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: vault-hermes-jenkins-token-seed + namespace: flux-system + annotations: + kustomize.toolkit.fluxcd.io/ssa: IfNotPresent +spec: + interval: 10m + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + path: ./services/vault-hermes-jenkins-token-seed + targetNamespace: vault + prune: true + wait: true + timeout: 5m + dependsOn: + - name: vault + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault diff --git a/clusters/atlas/flux-system/applications/vault/kustomization.yaml b/clusters/atlas/flux-system/applications/vault/kustomization.yaml index 7dd03da8..58f8ef7d 100644 --- a/clusters/atlas/flux-system/applications/vault/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vault/kustomization.yaml @@ -16,6 +16,11 @@ spec: targetNamespace: vault prune: true wait: true + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-k8s-auth-hermes-9 + namespace: vault dependsOn: - name: longhorn - name: helm diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index ce9ceb46..493c31a4 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -15,6 +15,10 @@ RUN apt-get update \ # Paid/provider-backed search remains selectable through normal Hermes config. RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4 +ARG HERMES_KANIKO_HEREDOC_COMPAT=0 +COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile +COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py + # Keep dashboard chat sockets tied to the intended React mount and conversation. # A resumed conversation needs a different PTY attachment key from a fresh chat; # reconnects to that same conversation must keep using the same key. @@ -252,6 +256,7 @@ for (const [before, after, label] of [ } fs.writeFileSync(sidebarPath, sidebar); NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac # The upstream OIDC gate authenticates users but deliberately treats the # dashboard as one shared workstation. Allow a deployment to narrow that @@ -324,6 +329,7 @@ for before, after, label in ( source = source.replace(before, after, 1) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac # Give trusted plugins a pre-turn routing hook. It runs after fallback runtime # restoration but before Hermes builds its provider-specific system prompt. @@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1: ) oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs. # Upstream currently applies only statically declared model_routes there, so @@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1) source = source[:runs_start] + runs_source path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac # ``create_task(initial_status="blocked")`` is an explicit operator park, but # upstream only treats later block_task() events as sticky. Recognize the @@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1: ) cli_path.write_text(cli_source.replace(help_before, help_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \ @@ -816,6 +825,7 @@ if source.count(before) != 1: ) path.write_text(source.replace(before, after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac # Keep the dashboard's account cards aligned with the managed provider lanes. # Switchyard receives Claude Code subscription OAuth through the environment; @@ -882,6 +892,7 @@ for before, after, label in replacements: path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac # Make parent-linked API workers first-class live dashboard sessions. They stay # active until the API runner closes them, even during a long final model call; @@ -975,6 +986,7 @@ source = source.replace(row_before, row_after) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac # Keep the browser terminal reliable across GPU context loss, make durable # worker lineage visible, expose an accessible API-worker transcript, and @@ -1543,9 +1555,12 @@ function RootRedirect() { fs.writeFileSync(path, source); } NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate +RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py + RUN cd /opt/hermes/web \ && npm run build \ && grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \ diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index 1327bda4..6c13b88f 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -5,3 +5,5 @@ !dockerfiles/hermes-public-extract/** !dockerfiles/hermes-session-activity-panel.tsx !dockerfiles/hermes-session-migrate.py +!dockerfiles/Dockerfile.hermes-agent +!dockerfiles/hermes-kaniko-heredoc-runner.py diff --git a/dockerfiles/hermes-kaniko-heredoc-runner.py b/dockerfiles/hermes-kaniko-heredoc-runner.py new file mode 100644 index 00000000..91a1330a --- /dev/null +++ b/dockerfiles/hermes-kaniko-heredoc-runner.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2. + +Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN +implementation executes only ``CmdLine``. The interpreters therefore receive +empty stdin and exit successfully. Docker and BuildKit execute these blocks +normally, so this compatibility runner is enabled only by the Kaniko pipeline. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path + + +MAX_DOCKERFILE_BYTES = 2_000_000 +BLOCKS = ( + ("RUN node <<'NODE'", "NODE", ("node",)), + ("RUN python - <<'PY'", "PY", ("python", "-")), +) +EXPECTED_COMMANDS = ("node", *("python",) * 7, "node") +PARSER_DIRECTIVE = re.compile( + r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE +) + + +def _escape_character(lines: list[str]) -> str: + """Return the one Dockerfile parser escape directive, or its default.""" + escape = "\\" + directives: set[str] = set() + for line in lines: + if not line.strip(): + break + match = PARSER_DIRECTIVE.fullmatch(line) + if not match: + break + name, value = match.group(1).lower(), match.group(2) + if name in directives: + raise ValueError(f"multiple Dockerfile {name} directives") + directives.add(name) + if name == "escape": + if value not in {"\\", "`"}: + raise ValueError("unsupported Dockerfile escape directive") + escape = value + return escape + + +def _continued(line: str, escape: str) -> bool: + """Match Docker's unescaped continuation marker at physical line end.""" + stripped = line.rstrip() + count = len(stripped) - len(stripped.rstrip(escape)) + return count % 2 == 1 + + +def _logical_instruction( + lines: list[str], index: int, escape: str +) -> tuple[str, int]: + """Normalize escape-newline pairs, including split opcodes/operators.""" + logical = lines[index] + while _continued(logical.split("\n")[-1], escape): + logical = logical.rstrip() + logical = logical[:-1] + index += 1 + while index < len(lines) and lines[index].lstrip().startswith("#"): + index += 1 + if index >= len(lines) or not lines[index].strip(): + raise ValueError("unterminated Dockerfile line continuation") + logical += lines[index].lstrip() + return logical, index + + +def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]: + """Inventory every RUN heredoc, then return only the reviewed contract.""" + markers = {start: (end, command) for start, end, command in BLOCKS} + lines = source.splitlines() + escape = _escape_character(lines) + blocks: list[tuple[tuple[str, ...], str]] = [] + index = 0 + while index < len(lines): + marker = markers.get(lines[index]) + if marker is None: + if not lines[index].strip() or lines[index].lstrip().startswith("#"): + index += 1 + continue + logical, index = _logical_instruction(lines, index, escape) + stripped = logical.lstrip() + if ( + stripped[:3].upper() == "RUN" + and (len(stripped) == 3 or stripped[3].isspace()) + and "<<" in stripped + ): + raise ValueError( + "unsupported RUN heredoc outside the exact reviewed contract" + ) + index += 1 + continue + end, command = marker + body_start = index + 1 + index = body_start + while index < len(lines) and lines[index] != end: + index += 1 + if index == len(lines): + raise ValueError(f"unterminated {command[0]} heredoc") + body = "\n".join(lines[body_start:index]) + "\n" + if not body.strip(): + raise ValueError(f"empty {command[0]} heredoc") + blocks.append((command, body)) + index += 1 + + commands = tuple(command[0] for command, _body in blocks) + if commands != EXPECTED_COMMANDS: + raise ValueError( + "Hermes Dockerfile heredoc contract changed: " + f"expected {EXPECTED_COMMANDS!r}, received {commands!r}" + ) + return blocks + + +def replay(dockerfile: Path, block_index: int) -> None: + """Execute one exact heredoc after inventorying the complete Dockerfile.""" + size = dockerfile.stat().st_size + if size < 1 or size > MAX_DOCKERFILE_BYTES: + raise ValueError("Hermes Dockerfile size is outside the reviewed boundary") + source = dockerfile.read_text(encoding="utf-8") + blocks = extract_blocks(source) + if block_index < 1 or block_index > len(blocks): + raise ValueError("heredoc block index is outside the reviewed contract") + command, body = blocks[block_index - 1] + print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}") + subprocess.run(command, input=body, text=True, check=True) + + +def main() -> int: + """Parse the one explicit Dockerfile path and replay its reviewed patches.""" + parser = argparse.ArgumentParser() + parser.add_argument("--dockerfile", required=True, type=Path) + parser.add_argument("--block-index", required=True, type=int) + args = parser.parse_args() + replay(args.dockerfile, args.block_index) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/harbor/hermes-agent-immutability-job.yaml b/services/harbor/hermes-agent-immutability-job.yaml new file mode 100644 index 00000000..5df877bb --- /dev/null +++ b/services/harbor/hermes-agent-immutability-job.yaml @@ -0,0 +1,79 @@ +# services/harbor/hermes-agent-immutability-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor +spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: ensure + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: https://registry.bstein.dev/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-agent-immutability-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 6a4ce002..7784bb50 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -12,9 +12,14 @@ resources: - certificate.yaml - helmrelease.yaml - vault-sync-deployment.yaml + - policy-bootstrap-serviceaccount.yaml + - hermes-agent-immutability-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: - name: harbor-vault-entrypoint files: - scripts/vault-entrypoint.sh + - name: harbor-hermes-agent-immutability-script + files: + - harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py diff --git a/services/harbor/policy-bootstrap-serviceaccount.yaml b/services/harbor/policy-bootstrap-serviceaccount.yaml new file mode 100644 index 00000000..4839196f --- /dev/null +++ b/services/harbor/policy-bootstrap-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/harbor/policy-bootstrap-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-policy-bootstrap + namespace: harbor +automountServiceAccountToken: true diff --git a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py new file mode 100644 index 00000000..a313cc80 --- /dev/null +++ b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Create and verify the narrowly scoped Hermes agent immutable-tag rule.""" + +from __future__ import annotations + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +PROJECT = "bstein" +REPOSITORY_PATTERN = "hermes-agent" +TAG_PATTERN = "git-*-build-*" +PUBLISH_ROBOT = "robot$jenkins-pipelines" +EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0" +MAX_RESPONSE = 1_048_576 +TRANSIENT_STATUSES = {429, 502, 503, 504} +EXPECTED_RULE = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": REPOSITORY_PATTERN, + } + ] + }, +} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Prevent Basic credentials from following an unexpected redirect.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +class HarborUnavailable(RuntimeError): + """Harbor is not ready yet, rather than returning a policy decision.""" + + +def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Return only the immutable contract fields Harbor must preserve.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in rule.get("tag_selectors") or [] + if isinstance(selector, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(selector, dict) + ] + }, + } + + +def targets_hermes_builds(rule: dict[str, Any]) -> bool: + """Detect a rule that claims this exact repository and tag selector.""" + normalized = normalized_rule(rule) + return ( + normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"] + and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"] + ) + + +class HarborClient: + """Bounded same-origin client for Harbor's immutable-tag API.""" + + def __init__(self, origin: str, username: str, password: str) -> None: + normalized_origin = origin.rstrip("/") + if normalized_origin != EXPECTED_ORIGIN: + raise ValueError("Harbor API origin is not the pinned production API") + self.origin = normalized_origin + token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + self.headers = {"Authorization": f"Basic {token}"} + self.opener = urllib.request.build_opener(NoRedirect()) + + def request( + self, method: str, path: str, payload: dict[str, Any] | None = None + ) -> tuple[int, bytes, dict[str, str]]: + """Issue one request, returning even non-2xx responses for strict checks.""" + data = None + headers = dict(self.headers) + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode() + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.origin}{path}", data=data, headers=headers, method=method + ) + try: + response = self.opener.open(request, timeout=20) + except urllib.error.HTTPError as exc: + response = exc + except (urllib.error.URLError, TimeoutError) as exc: + raise HarborUnavailable("Harbor policy API is unavailable") from exc + with response: + body = response.read(MAX_RESPONSE + 1) + if len(body) > MAX_RESPONSE: + raise RuntimeError("Harbor response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def list_rules(client: HarborClient) -> list[dict[str, Any]]: + """Read and validate the complete small rule set for the project.""" + path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100" + status, body, headers = client.request("GET", path) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}") + values = _json_list(body, "immutable rule") + _require_complete_page(values, headers, "immutable rule") + return values + + +def _json_list(body: bytes, label: str) -> list[dict[str, Any]]: + """Decode one bounded Harbor list without accepting a partial shape.""" + try: + value = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise RuntimeError(f"Harbor {label} list has an invalid shape") + return value + + +def _require_complete_page( + values: list[dict[str, Any]], headers: dict[str, str], label: str +) -> None: + """Reject a truncated first page or a proxy that strips count evidence.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError(f"Harbor {label} list omitted a valid total count") + if int(raw_total) != len(values): + raise RuntimeError(f"Harbor {label} list was truncated") + + +def _allowed(access: dict[str, Any], resource: str, action: str) -> bool: + """Match one positive robot permission, accepting Harbor's default effect.""" + return ( + access.get("resource") == resource + and access.get("action") == action + and access.get("effect") in (None, "", "allow") + ) + + +def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return the existing bstein scope after validating its push boundary.""" + permissions = robot.get("permissions") + if not isinstance(permissions, list) or not all( + isinstance(item, dict) for item in permissions + ): + raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape") + matches = [ + item + for item in permissions + if item.get("kind") == "project" and item.get("namespace") == PROJECT + ] + if len(matches) != 1: + raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope") + access = matches[0].get("access") + if not isinstance(access, list) or not all(isinstance(item, dict) for item in access): + raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape") + for required in (("repository", "pull"), ("repository", "push")): + if not any(_allowed(item, *required) for item in access): + raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary") + immutable_access = [ + item for item in access if item.get("resource") == "immutable-tag" + ] + if immutable_access and not ( + len(immutable_access) == 1 + and _allowed(immutable_access[0], "immutable-tag", "list") + ): + raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access") + return matches[0], access + + +def _verified_publisher(robot: dict[str, Any]) -> bool: + """Check the exact identity and least-privilege policy after persistence.""" + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + return False + try: + _scope, access = _publisher_scope(robot) + except RuntimeError: + return False + return any(_allowed(item, "immutable-tag", "list") for item in access) + + +def ensure_publisher_can_read_rule(client: HarborClient) -> int: + """Grant only immutable-tag:list to the existing Jenkins push robot.""" + status, body, headers = client.request("GET", "/robots?page=1&page_size=100") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot list returned HTTP {status}") + robots = _json_list(body, "robot") + _require_complete_page(robots, headers, "robot") + matches = [ + item + for item in robots + if item.get("name") == PUBLISH_ROBOT + ] + if len(matches) != 1: + raise RuntimeError("expected exactly one Jenkins Harbor publisher robot") + robot_id = matches[0].get("id") + if not isinstance(robot_id, int) or robot_id < 1: + raise RuntimeError("Jenkins Harbor robot omitted a valid ID") + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot read returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot read returned HTTP {status}") + try: + robot = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid robot JSON") from exc + if not isinstance(robot, dict): + raise RuntimeError("Harbor robot response has an invalid shape") + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + raise RuntimeError("Jenkins Harbor publisher identity is not active and exact") + scope, access = _publisher_scope(robot) + if any(_allowed(item, "immutable-tag", "list") for item in access): + return robot_id + + # Harbor updates permissions through PUT /robots/{id}; its separate PATCH + # endpoint is the only secret-rotation operation. Preserve every current + # scope and field while adding the one read-only action. + access.append({"resource": "immutable-tag", "action": "list"}) + payload = { + "name": robot["name"], + "description": robot.get("description") or "", + "level": robot["level"], + "disable": robot["disable"], + "permissions": robot["permissions"], + } + duration = robot.get("duration") + if duration is not None: + if not isinstance(duration, int): + raise RuntimeError("Jenkins Harbor robot duration has an invalid shape") + payload["duration"] = duration + # Keep the validated object reference live in the preserved permission list. + if scope.get("access") is not access: + raise RuntimeError("Jenkins Harbor robot permission normalization drifted") + status, _body, _headers = client.request( + "PUT", f"/robots/{robot_id}", payload + ) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot update returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot policy update returned HTTP {status}") + + for attempt in range(1, 6): + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status == 200: + try: + persisted = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + persisted = None + if isinstance(persisted, dict) and _verified_publisher(persisted): + return robot_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("Jenkins Harbor robot policy did not verify exactly") + + +def ensure_rule(client: HarborClient) -> int: + """Create once, or validate the one exact enabled rule already present.""" + rules = list_rules(client) + matches = [rule for rule in rules if targets_hermes_builds(rule)] + if len(matches) > 1: + raise RuntimeError("multiple Hermes agent immutable rules exist") + if matches: + if normalized_rule(matches[0]) != EXPECTED_RULE: + raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact") + rule_id = matches[0].get("id") + if not isinstance(rule_id, int) or rule_id < 1: + raise RuntimeError("Harbor immutable rule omitted a valid ID") + return rule_id + + path = f"/projects/{PROJECT}/immutabletagrules" + status, _body, headers = client.request("POST", path, EXPECTED_RULE) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}") + if status != 201: + raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}") + location = headers.get("Location") or headers.get("location") or "" + api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/") + expected_prefix = f"{api_path}{path}/" + if not location.startswith(expected_prefix): + raise RuntimeError("Harbor immutable rule create omitted the exact Location") + suffix = location[len(expected_prefix) :] + if not suffix.isdecimal() or int(suffix) < 1: + raise RuntimeError("Harbor immutable rule Location has an invalid ID") + + # Harbor persists synchronously, but bounded retries distinguish a stale + # read from accepting an unverified policy. + for attempt in range(1, 6): + matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)] + if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE: + rule_id = matches[0].get("id") + if rule_id == int(suffix): + return rule_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("created Harbor immutable rule did not verify exactly") + + +def main() -> int: + """Load the runtime-only admin credential and enforce the tracked policy.""" + origin = os.environ.get("HARBOR_API_ORIGIN", "") + password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", "")) + password = password_file.read_text(encoding="utf-8").strip() + if not password: + raise RuntimeError("Harbor admin password is empty") + client = HarborClient(origin, "admin", password) + for attempt in range(1, 13): + try: + rule_id = ensure_rule(client) + robot_id = ensure_publisher_can_read_rule(client) + break + except HarborUnavailable: + if attempt == 12: + raise + time.sleep(min(attempt * 2, 15)) + print( + "Hermes agent immutable build-tag rule is active " + f"(id={rule_id}); publisher preflight access is active (robot={robot_id})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..aec53d0c 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report must say `retained Ariadne evidence` when that fallback is used; it must not pretend direct Jenkins access succeeded. +## Publishing an agent image after review + +The `hermes-agent-image` Jenkins job is the only supported agent image builder. +It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder` +ServiceAccount without a service-account token, host socket, +privileged container, or writable Git credential. It accepts only an exact +40-character revision that is both the checked-out commit and current +`atlas/titan-iac` `main`, so a human must merge the source PR first. + +The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum +scheduler preference. Healthy rpi4 workers remain valid fallback capacity when +the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and +reserved last-resort nodes remain excluded by the required placement boundary. + +Kaniko runs as UID 0 because it must unpack an image root filesystem and enter +that filesystem for Dockerfile `RUN` instructions. Its capability set is the +minimum proven by an exact ARM64 no-push build of this Dockerfile: +`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required +because pinned Kaniko applies the base image's supplementary group list before +each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account +while downloading package indexes. It still has no privilege +escalation, service-account token, host path, daemon socket, or Docker/BuildKit +TCP endpoint and uses the runtime-default seccomp profile. This is residual +root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a +human-reviewed `main` revision; Dockerfile changes require the same scrutiny as +executable cluster code. + +Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize +their inline files when executing a command. The Jenkins lane therefore enables +a bounded compatibility replay after copying the reviewed Dockerfile and runner +from the same checked-out commit. Before every replay, the runner inventories +the entire Dockerfile as logical instructions, including split opcodes, +operators, and either Dockerfile escape character. It rejects every heredoc +form outside the exact nine Node/Python blocks before launching an interpreter. +Each accepted body is then replayed immediately after its corresponding Kaniko +`RUN`, preserving source order before any dependent regression. Docker and +BuildKit keep their native behavior because the compatibility argument defaults +off. The final TypeScript build, Python compile, and source assertions remain +mandatory, so an omitted or drifted replay fails before any image can pass +release verification. + +From agent.hermes, trigger that one fixed job with: + +```sh +jenkins_image_build_trigger.py '' +``` + +The helper has no general Jenkins credential or caller-selectable job name. Its +Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job +also requires its fixed publish confirmation. Each run claims the unique tag +`git--build-` and refuses to overwrite an +existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then +independently reads the pushed tag from Harbor before archiving a JSON record +and Flux digest patch. A successful post-condition then revalidates the exact +six-file evidence set and archives those six explicit paths with empty archives +forbidden. Apply that patch on a new branch and submit it for human review; the +build never changes Git, reconciles Flux, or deploys by itself. +The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts +only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact +same-origin `/queue/item//` location. An unauthenticated request is +denied, and the token cannot select, configure, read, or administer another +Jenkins job. The pipeline independently rejects any commit that is not the +current `origin/main`, preserving the human merge/review boundary. + +Harbor independently enforces an enabled immutable-tag rule scoped to only the +`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT, +retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins +preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact +rule with a runtime-only Vault credential; it refuses to alter a conflicting +rule and retries only explicit transport/readiness failures while Harbor +starts. The same Job grants the existing Jenkins publisher only +`immutable-tag:list` on `bstein`, preserving its other project scopes and never +calling Harbor's separate secret-rotation endpoint. The pipeline uses that +read-only permission to require the exact enabled rule before Kaniko starts. +That check also makes an already-running Jenkins controller fail closed during +rollout; Jenkins cannot become Ready on the new revision before the policy Job +succeeds. + +The tracked rollout order is deliberate: the revisioned Vault role Job must +complete before the Vault Kustomization becomes Ready. A separate Flux +Kustomization then runs the seed Job under the dedicated +`hermes-jenkins-token-seed` identity, which can only create/read the exact +`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins +depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins. +The seeder uses KV-v2 CAS create-only semantics. It never changes an existing +token, never fills a missing field in an existing secret, and fails closed when +it cannot distinguish absence from a read error. The recurring Vault +configuration job reconciles the same narrow role and policy. + +Rotate this fixed-job token only as a coordinated operator action: + +1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to + drain. +2. Read the current KV-v2 metadata version for + `kv/atlas/hermes/developer-jenkins`. +3. Generate a fresh value from Vault and patch only `build_token` with + `vault kv patch -cas=`. Keep the value in a mode-0600 + temporary file or standard input, never a command argument or log. +4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and + wait until it is Ready. Do not resume triggers between those two consumers. +5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch + with the previous value as another coordinated rotation; never delete the + secret to make the seeder recreate it. + ## The actual supervised triage algorithm 1. Classify the request as test/build triage, service health, or alert tuning. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..028c5d70 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -56,6 +56,11 @@ spec: {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} {{ .Data.data.username }} {{- end }} + vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins + vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: | + {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}} + {{ .Data.data.build_token }} + {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..6009cd7b 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -77,6 +77,7 @@ configMapGenerator: - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py + - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py - kanban_status_recovery.py=scripts/kanban_status_recovery.py - migrate_herdr_state.py=scripts/migrate_herdr_state.py - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py new file mode 100755 index 00000000..6cd91ed8 --- /dev/null +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Trigger only the reviewed-main Hermes agent image release job.""" + +from __future__ import annotations + +import argparse +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +JENKINS_ORIGIN = "https://ci.bstein.dev" +JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters" +JOB_NAME = "hermes-agent-image" +TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$") + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep a queued-build redirect from becoming an unauthorized job read.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _open_without_redirect(request: urllib.request.Request, timeout: int): + """Return the Build Token Root response, including its expected HTTP 303.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + if exc.code == 303: + return exc + raise + + +def trigger_build( + revision: str, + *, + token_file: Path = TOKEN_FILE, + opener=_open_without_redirect, +) -> dict[str, str | int]: + """Post the fixed job parameters using its job-scoped build token.""" + revision = revision.strip() + if not REVISION_PATTERN.fullmatch(revision): + raise ValueError("revision must be a lowercase full 40-character commit") + token = token_file.read_text(encoding="utf-8").strip() + if not token: + raise RuntimeError("Jenkins image-build token is empty") + payload = urllib.parse.urlencode( + { + "job": JOB_NAME, + "token": token, + "PUBLISH_IMAGE": "true", + "EXPECTED_SOURCE_REVISION": revision, + "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT", + } + ).encode("utf-8") + request = urllib.request.Request( + JENKINS_BUILD_URL, + data=payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + with opener(request, timeout=20) as response: + status = int(response.status) + location = response.headers.get("Location", "") + if status not in {201, 303}: + raise RuntimeError(f"Jenkins trigger returned HTTP {status}") + if not location: + raise RuntimeError("Jenkins trigger omitted the queue Location") + queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location) + parsed_queue = urllib.parse.urlsplit(queue_url) + expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN) + if ( + parsed_queue.scheme != expected_origin.scheme + or parsed_queue.netloc != expected_origin.netloc + or parsed_queue.query + or parsed_queue.fragment + or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path) + ): + raise RuntimeError("Jenkins returned an invalid queue Location") + # Never return the submitted URL: its form body contains the job token. + queue_path = parsed_queue.path + return { + "job": JOB_NAME, + "queue_path": queue_path, + "source_revision": revision, + "status": status, + } + + +def main() -> int: + """Validate one revision, trigger the bounded job, and print safe metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("revision", help="reviewed full commit currently on main") + args = parser.parse_args() + try: + result = trigger_build(args.revision) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..e8276a5b 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -95,6 +95,7 @@ def stage_agent() -> None: "chat-relay-key", "gitea-token", "gitea-username", + "jenkins-image-build-token", "node-ssh-private-key", "node-ssh-config", "node-ssh-known-hosts", diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 48143682..456d0a1a 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -27,7 +27,7 @@ spec: prometheus.io/port: "9005" prometheus.io/path: /metrics vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/role: hermes-switchyard vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram vault.hashicorp.com/agent-inject-template-relay-key: | {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index e9aa57b9..2a1e81eb 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -652,6 +652,25 @@ data: } } } + pipelineJob('hermes-agent-image') { + disabled(false) + description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/atlas/titan-iac.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('ci/Jenkinsfile.hermes-agent-image') + } + } + } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource { diff --git a/services/jenkins/configmap-plugins.yaml b/services/jenkins/configmap-plugins.yaml index 049bc1bd..ea4b450d 100644 --- a/services/jenkins/configmap-plugins.yaml +++ b/services/jenkins/configmap-plugins.yaml @@ -20,6 +20,7 @@ data: gitea:268.v75e47974c01d gitea-checks:603.621.vc708da_fb_371d multibranch-scan-webhook-trigger:1.0.11 + build-token-root:365.v717f8685a_09e # Structured test evidence. Without junit the `junit` step throws # NoSuchMethodError, jenkins.failed_tests is always empty, and triage # has only raw console text to work from. Pinned to the newest release diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 1fa22374..46109da7 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -65,6 +65,9 @@ spec: ARIADNE_JENKINS_API_USER={{ .Data.data.username }} ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }} {{ end }} + {{ with secret "kv/data/atlas/hermes/developer-jenkins" }} + HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }} + {{ end }} bstein.dev/restarted-at: "2026-05-20T09:40:31Z" spec: serviceAccountName: jenkins diff --git a/services/jenkins/hermes-image-builder-serviceaccount.yaml b/services/jenkins/hermes-image-builder-serviceaccount.yaml new file mode 100644 index 00000000..98279417 --- /dev/null +++ b/services/jenkins/hermes-image-builder-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/jenkins/hermes-image-builder-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-image-builder + namespace: jenkins +automountServiceAccountToken: false diff --git a/services/jenkins/kustomization.yaml b/services/jenkins/kustomization.yaml index b69d7642..5f3e0554 100644 --- a/services/jenkins/kustomization.yaml +++ b/services/jenkins/kustomization.yaml @@ -5,6 +5,7 @@ namespace: jenkins resources: - namespace.yaml - serviceaccount.yaml + - hermes-image-builder-serviceaccount.yaml - vault-serviceaccount.yaml - pvc.yaml - cache-pvc.yaml diff --git a/services/jenkins/secretproviderclass.yaml b/services/jenkins/secretproviderclass.yaml index a9d9dd50..148f193f 100644 --- a/services/jenkins/secretproviderclass.yaml +++ b/services/jenkins/secretproviderclass.yaml @@ -8,7 +8,7 @@ spec: provider: vault parameters: vaultAddress: "http://vault.vault.svc.cluster.local:8200" - roleName: "jenkins" + roleName: "jenkins-vault-sync" objects: | - objectName: "harbor-pull__dockerconfigjson" secretPath: "kv/data/atlas/shared/harbor-pull" diff --git a/services/vault-hermes-jenkins-token-seed/job.yaml b/services/vault-hermes-jenkins-token-seed/job.yaml new file mode 100644 index 00000000..17f7a4ca --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/job.yaml @@ -0,0 +1,68 @@ +# services/vault-hermes-jenkins-token-seed/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: hermes-jenkins-token-seed + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: seed + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 + imagePullPolicy: IfNotPresent + command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh] + env: + - name: HOME + value: /tmp + - name: VAULT_ADDR + value: http://vault.vault.svc.cluster.local:8200 + - name: VAULT_K8S_ROLE + value: hermes-jenkins-token-seed + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: vault-hermes-jenkins-token-seed-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/vault-hermes-jenkins-token-seed/kustomization.yaml b/services/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..920caa50 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,13 @@ +# services/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: vault +resources: + - serviceaccount.yaml + - job.yaml +generatorOptions: + disableNameSuffixHash: true +configMapGenerator: + - name: vault-hermes-jenkins-token-seed-script + files: + - vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh diff --git a/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh new file mode 100644 index 00000000..1514b248 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env sh +set -eu + +secret_path="kv/atlas/hermes/developer-jenkins" +secret_api_path="kv/data/atlas/hermes/developer-jenkins" +jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}" +payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json" + +log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; } +cleanup() { rm -f "${payload_file}"; } +trap cleanup EXIT HUP INT TERM + +retry_command() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + command_output="$("$@" 2>&1)" + command_status=$? + set -e + if [ "${command_status}" -eq 0 ]; then + printf '%s' "${command_output}" + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + return "${command_status}" +} + +ensure_token() { + if [ -n "${VAULT_TOKEN:-}" ]; then + return + fi + jwt="$(cat "${jwt_file}")" + VAULT_TOKEN="$(retry_command vault write -field=token \ + auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || { + log "Vault Kubernetes login failed after retries" + exit 1 + } + unset jwt + if [ -z "${VAULT_TOKEN}" ]; then + log "Vault Kubernetes login returned an empty token" + exit 1 + fi + export VAULT_TOKEN +} + +# Return 0 for the complete field, 10 when absent, 11 when present but +# incomplete, and 12 for a persistent authorization/transport failure. +read_build_token() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)" + secret_status=$? + set -e + if [ "${secret_status}" -eq 0 ]; then + unset secret_json + set +e + build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)" + field_status=$? + set -e + if [ "${field_status}" -eq 0 ]; then + case "${build_token}" in + ''|*[!0-9a-f]*) unset build_token; return 11 ;; + esac + if [ "${#build_token}" -eq 64 ]; then + unset build_token + return 0 + fi + unset build_token + return 11 + fi + if printf '%s' "${build_token}" | grep -q 'No value found'; then + unset build_token + return 11 + fi + unset build_token + elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then + unset secret_json + return 10 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + unset secret_json + return 12 +} + +ensure_token + +if read_build_token; then + read_status=0 +else + read_status=$? +fi +case "${read_status}" in + 0) + log "job-scoped token already present; no write performed" + exit 0 + ;; + 11) + log "secret exists without a valid build_token; refusing to overwrite existing fields" + exit 1 + ;; + 12) + log "secret read failed after retries; refusing to seed" + exit 1 + ;; + 10) ;; + *) + log "unexpected secret read state ${read_status}" + exit 1 + ;; +esac + +new_token="$(retry_command vault write -field=random_bytes \ + sys/tools/random/32 format=hex)" || { + log "Vault random token generation failed after retries" + exit 1 + } +case "${new_token}" in + ''|*[!0-9a-f]*) + unset new_token + log "Vault returned an invalid random token" + exit 1 + ;; +esac +if [ "${#new_token}" -ne 64 ]; then + unset new_token + log "Vault returned an invalid random token" + exit 1 +fi + +umask 077 +printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \ + "${new_token}" > "${payload_file}" +unset new_token + +set +e +create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)" +create_status=$? +set -e +cleanup +if [ "${create_status}" -eq 0 ]; then + log "job-scoped token created with KV-v2 CAS create-only semantics" + exit 0 +fi + +if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then + unset create_error + if read_build_token; then + log "another seeder won CAS; existing token preserved" + exit 0 + fi + log "CAS race did not produce a readable build_token; refusing to continue" + exit 1 +fi + +unset create_error +log "KV-v2 CAS create failed; refusing to retry a write" +exit 1 diff --git a/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml new file mode 100644 index 00000000..962471bd --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-jenkins-token-seed + namespace: vault +automountServiceAccountToken: true diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..4bcee103 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-9 namespace: vault spec: backoffLimit: 2 @@ -15,16 +15,26 @@ spec: hardware: rpi5 kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] containers: - name: configure-k8s-auth - image: hashicorp/vault:1.21.4 + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 imagePullPolicy: IfNotPresent command: - sh - /scripts/vault_k8s_auth_configure.sh env: + - name: HOME + value: /tmp - name: VAULT_ADDR - value: http://10.43.57.249:8200 + value: http://vault.vault.svc.cluster.local:8200 - name: VAULT_K8S_ROLE value: vault-admin - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE @@ -38,6 +48,18 @@ spec: - name: token-reviewer mountPath: /var/run/secrets/vault-token-reviewer readOnly: true + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault resources: requests: cpu: 25m @@ -53,3 +75,5 @@ spec: - name: token-reviewer secret: secretName: vault-admin-token-reviewer + - name: tmp + emptyDir: {} diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..48e839e6 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -235,12 +235,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \ "mailu/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \ "harbor/* shared/harbor-pull" "hermes/developer-harbor" +harbor_policy_bootstrap_policy=' +path "kv/data/atlas/harbor/harbor-core" { + capabilities = ["read"] +} +' +write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}" +log "writing role harbor-policy-bootstrap" +vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \ + bound_service_account_names="harbor-policy-bootstrap" \ + bound_service_account_namespaces="harbor" \ + policies="harbor-policy-bootstrap" \ + ttl="${role_ttl}" write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \ "nextcloud/* shared/keycloak-admin shared/postmark-relay" "" write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \ "comms/* shared/chat-ai-keys-runtime shared/harbor-pull" "" -write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \ - "jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins" +write_policy_and_role "jenkins" "jenkins" "jenkins" \ + "jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" "" +write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \ + "shared/harbor-pull" "" write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \ "monitoring/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "logging" "logging" "logging-vault-sync" \ @@ -255,8 +269,25 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" -write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ +write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \ + "hermes/chat-telegram" "" +hermes_jenkins_token_seed_policy=' +path "kv/data/atlas/hermes/developer-jenkins" { + capabilities = ["create", "read"] +} +path "sys/tools/random/32" { + capabilities = ["update"] +} +' +write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}" +log "writing role hermes-jenkins-token-seed" +vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \ + bound_service_account_names="hermes-jenkins-token-seed" \ + bound_service_account_namespaces="vault" \ + policies="hermes-jenkins-token-seed" \ + ttl="${role_ttl}" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..73d68fa2 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,10 @@ } ], "managed_modules": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -35,6 +39,10 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -57,7 +65,10 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "dockerfiles", "scripts.render.dashboards_render_atlas", + "services/harbor/scripts", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -168,6 +179,10 @@ "coverage": { "minimum_percent": 95.0, "tracked_files": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py new file mode 100644 index 00000000..8d7418d3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder.py @@ -0,0 +1,473 @@ +"""Safety and artifact contracts for the Hermes agent image release lane.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import re +import urllib.parse +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" +HEREDOC_RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" + + +def _load_release_module(): + spec = importlib.util.spec_from_file_location( + "hermes_image_release", RELEASE_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_trigger_module(): + spec = importlib.util.spec_from_file_location( + "jenkins_image_build_trigger", TRIGGER_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_heredoc_runner(): + spec = importlib.util.spec_from_file_location( + "hermes_kaniko_heredoc_runner", HEREDOC_RUNNER + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE_PATH.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_pod_is_daemonless_and_kernel_bounded() -> None: + """The builder gets only proven build caps, never host, daemon, or K8s access.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + assert "hostPath" not in source + assert "docker.sock" not in source + assert "tcp://" not in source + assert "buildkitd" not in source.lower() + assert "dind" not in source.lower() + assert "serviceAccountToken" not in source + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + assert kaniko["securityContext"]["capabilities"]["add"] == [ + "CHOWN", + "FOWNER", + "DAC_OVERRIDE", + "SETGID", + "SETUID", + ] + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + assert "add" not in containers["jnlp"]["securityContext"]["capabilities"] + assert "add" not in containers["python"]["securityContext"]["capabilities"] + + +def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: + """Publishing requires explicit confirmation and a reviewed main commit.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + assert 'test "${PUBLISH_IMAGE}" = "true"' in source + assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source + assert "git rev-parse origin/main" in source + assert "credentialsId: 'harbor-robot'" in source + assert "set +x" in source + assert "umask 077" in source + assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source + assert "/busybox/rm -f /kaniko/.docker/config.json" in source + assert "--digest-file=" in source + assert "--image-name-tag-with-digest-file=" in source + assert "--destination=" in source + assert "assert-absent" in source + assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source + assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None: + """Pinned Kaniko's ignored inline files are replayed in exact source order.""" + module = _load_heredoc_runner() + dockerfile = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + blocks = module.extract_blocks(dockerfile) + assert tuple(command[0] for command, _body in blocks) == ( + "node", + "python", + "python", + "python", + "python", + "python", + "python", + "python", + "node", + ) + assert "session message total" in blocks[-1][1] + compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1] + assert compat.count("python /tmp/hermes-kaniko-heredoc-runner.py") == 9 + assert [int(value) for value in re.findall(r"--block-index (\d+)", compat)] == list( + range(1, 10) + ) + ignored = ( + REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore" + ).read_text() + assert "!dockerfiles/Dockerfile.hermes-agent" in ignored + assert "!dockerfiles/hermes-kaniko-heredoc-runner.py" in ignored + + +def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Missing blocks fail closed and each reviewed body gets a fresh process.""" + module = _load_heredoc_runner() + source = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + with pytest.raises(ValueError, match="contract changed"): + module.extract_blocks(source.replace("RUN node <<'NODE'", "RUN node", 1)) + + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + + monkeypatch.setattr(module.subprocess, "run", run) + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text(source, encoding="utf-8") + for block_index in range(1, 10): + module.replay(dockerfile, block_index) + assert len(calls) == 9 + assert all(call[1]["check"] is True for call in calls) + assert all(call[1]["text"] is True for call in calls) + assert all(call[1]["input"].endswith("\n") for call in calls) + + +def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None: + """JCasC must not publish unreviewed branch contents or poll automatically.""" + config = yaml.safe_load( + (REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text( + encoding="utf-8" + ) + ) + jobs = config["data"]["jobs.yaml"] + block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split( + "pipelineJob(", 1 + )[0] + assert "branches('*/main')" in block + assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block + assert ( + "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + ) + assert "pipelineTriggers" not in block + assert "scmTrigger" not in block + + +def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None: + """Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API.""" + module = _load_trigger_module() + revision = "a" * 40 + token_path = tmp_path / "token" + token_path.write_text("private-job-token\n", encoding="utf-8") + captured = {} + + class Response(io.BytesIO): + status = 201 + headers = {"Location": "https://ci.bstein.dev/queue/item/42/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def opener(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(b"") + + result = module.trigger_build(revision, token_file=token_path, opener=opener) + request = captured["request"] + fields = urllib.parse.parse_qs(request.data.decode("utf-8")) + assert request.full_url == module.JENKINS_BUILD_URL + assert fields == { + "CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"], + "EXPECTED_SOURCE_REVISION": [revision], + "PUBLISH_IMAGE": ["true"], + "job": ["hermes-agent-image"], + "token": ["private-job-token"], + } + assert captured["timeout"] == 20 + assert "private-job-token" not in json.dumps(result) + assert result["source_revision"] == revision + + +def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None: + """No user-controlled job, URL, or abbreviated revision reaches Jenkins.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + with pytest.raises(ValueError): + module.trigger_build("main", token_file=token_path) + token_path.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.trigger_build("a" * 40, token_file=token_path) + + +def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None: + """HTTP 303 means the exact release is already queued, not a trigger failure.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + + class Response(io.BytesIO): + status = 303 + headers = {"Location": "https://ci.bstein.dev/queue/item/7/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + result = module.trigger_build( + "e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response() + ) + assert result["status"] == 303 + assert result["queue_path"] == "/queue/item/7/" + + +def test_job_token_is_generated_and_injected_only_at_runtime() -> None: + """The fixed-job credential stays in Vault and pod-lifetime memory.""" + plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text( + encoding="utf-8" + ) + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + seeder = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" + ).read_text(encoding="utf-8") + jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text( + encoding="utf-8" + ) + agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text( + encoding="utf-8" + ) + stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text( + encoding="utf-8" + ) + assert "build-token-root:365.v717f8685a_09e" in plugins + assert 'write_raw_policy "hermes-jenkins-token-seed"' in vault + assert "sys/tools/random/32 format=hex" in seeder + assert '"options":{"cas":0}' in seeder + assert "kv/data/atlas/hermes/developer-jenkins" in seeder + assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins + assert "agent-inject-secret-jenkins-image-build-token" in agent + assert '"jenkins-image-build-token"' in stage + + +def test_release_renderer_preserves_manifest_and_emits_safe_artifacts( + tmp_path: Path, +) -> None: + """The release artifact is exact, reviewable, and contains no credentials.""" + module = _load_release_module() + old_digest = "sha256:" + "1" * 64 + new_digest = "sha256:" + "2" * 64 + revision = "a" * 40 + build_number = "17" + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + f" digest: {old_digest}\n", + encoding="utf-8", + ) + output = tmp_path / "out" + metadata = module.write_release_artifacts( + digest=f"{new_digest}\n", + source_revision=revision, + build_number=build_number, + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}", + kustomization=manifest, + output_dir=output, + ) + + assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n") + assert ( + (output / "hermes-kustomization.yaml") + .read_text(encoding="utf-8") + .endswith(f"{new_digest}\n") + ) + patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8") + assert f"- digest: {old_digest}" in patch + assert f"+ digest: {new_digest}" in patch + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8")) + == metadata + ) + assert set(metadata) == { + "digest", + "build_number", + "flux_image", + "image", + "published_tag", + "source_revision", + } + + +@pytest.mark.parametrize( + ("digest", "revision", "destination"), + [ + ( + "latest", + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1", + ), + ( + "sha256:" + "1" * 64, + "short", + "registry.bstein.dev/bstein/hermes-agent:git-short-build-1", + ), + ( + "sha256:" + "1" * 64, + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:latest", + ), + ], +) +def test_release_renderer_rejects_unpinned_inputs( + tmp_path: Path, digest: str, revision: str, destination: str +) -> None: + """Only exact digests and immutable source-derived tags are accepted.""" + module = _load_release_module() + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + with pytest.raises(ValueError): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=destination, + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None: + """Missing, duplicate, or already-current image entries require human review.""" + module = _load_release_module() + digest = "sha256:" + "f" * 64 + missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n" + with pytest.raises(ValueError, match="found 0"): + module.render_kustomization(missing, digest) + + duplicate = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + ) + with pytest.raises(ValueError, match="found 2"): + module.render_kustomization(duplicate, digest) + + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n", + encoding="utf-8", + ) + revision = "b" * 40 + with pytest.raises(ValueError, match="already matches"): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1", + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_cli_reads_digest_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pipeline CLI uses the Kaniko digest file as its sole digest input.""" + module = _load_release_module() + digest = "sha256:" + "c" * 64 + revision = "d" * 40 + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:" + + "0" * 64 + + "\n", + encoding="utf-8", + ) + digest_file = tmp_path / "digest" + digest_file.write_text(digest + "\n", encoding="utf-8") + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" + image_file = tmp_path / "image" + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + output = tmp_path / "out" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "secret") + monkeypatch.setattr( + module, "verify_registry_digest", lambda *_args, **_kwargs: None + ) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "render", + "--digest-file", + str(digest_file), + "--image-file", + str(image_file), + "--source-revision", + revision, + "--build-number", + "9", + "--destination", + destination, + "--kustomization", + str(manifest), + "--output-dir", + str(output), + ], + ) + assert module.main() == 0 + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[ + "digest" + ] + == digest + ) diff --git a/testing/tests/test_hermes_image_builder_adversarial.py b/testing/tests/test_hermes_image_builder_adversarial.py new file mode 100644 index 00000000..be65e1de --- /dev/null +++ b/testing/tests/test_hermes_image_builder_adversarial.py @@ -0,0 +1,469 @@ +"""Adversarial boundaries for the Hermes image publisher and trigger.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Small context-managed HTTP response fixture.""" + + def __init__( + self, + status: int, + headers: dict[str, str] | None = None, + body: bytes = b"", + ): + super().__init__(body) + self.status = status + self.headers = headers or {} + + +def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None: + """Both Kaniko artifacts must describe one exact non-replayable tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence") + revision = "a" * 40 + digest = "sha256:" + "b" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42" + assert module.validate_destination(destination, revision, "42") == ( + revision, + "42", + ) + assert ( + module.validate_kaniko_evidence( + digest_text=f"{digest}\n", + image_text=f"{destination}@{digest}\n", + destination=destination, + ) + == digest + ) + + +@pytest.mark.parametrize( + ("digest_text", "image_template"), + [ + ("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"), + ("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64), + ("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"), + ("sha256:" + "a" * 64, "{destination}@{digest}\nextra"), + ], +) +def test_kaniko_evidence_rejects_cross_artifact_mismatch( + digest_text: str, image_template: str +) -> None: + """A digest, tag, repository, or cardinality mismatch stops rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch") + revision = "c" * 40 + digest = "sha256:" + "a" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8" + image_text = image_template.format(destination=destination, digest=digest) + with pytest.raises(ValueError): + module.validate_kaniko_evidence( + digest_text=digest_text, + image_text=image_text, + destination=destination, + ) + + +def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None: + """Only an authenticated 404 permits Kaniko to claim the unique tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight") + destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3" + captured = {} + + def missing(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(404) + + module.assert_tag_absent( + destination, username="robot", password="private", opener=missing + ) + request = captured["request"] + assert request.method == "GET" + assert request.full_url.startswith( + "https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/" + "hermes-agent/artifacts/" + ) + assert "private" not in request.full_url + assert captured["timeout"] == 20 + + for status, message in ( + (200, "already exists"), + (401, "HTTP 401"), + (503, "HTTP 503"), + ): + with pytest.raises(RuntimeError, match=message): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args, code=status: Response(code), + ) + + +def test_registry_preflight_requires_exact_server_immutable_policy() -> None: + """The already-running controller cannot publish before Flux installs the rule.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_policy_preflight") + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + captured = {} + + def exact(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response( + 200, + {"X-Total-Count": "1"}, + body=json.dumps([{"id": 9, **expected}]).encode(), + ) + + module.verify_immutable_policy(username="robot", password="private", opener=exact) + assert captured["url"].endswith( + "/projects/bstein/immutabletagrules?page=1&page_size=100" + ) + assert captured["timeout"] == 20 + + for status, body, headers, message in ( + (403, b"", {}, "HTTP 403"), + (200, b"[]", {"X-Total-Count": "0"}, "absent"), + ( + 200, + json.dumps([{**expected, "disabled": True}]).encode(), + {"X-Total-Count": "1"}, + "not exact", + ), + (200, b"not-json", {}, "invalid"), + ( + 200, + json.dumps([expected]).encode(), + {"X-Total-Count": "2"}, + "truncated", + ), + ): + with pytest.raises(RuntimeError, match=message): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, code=status, value=body, evidence=headers: Response( + code, evidence, body=value + ), + ) + + +def test_harbor_client_fails_closed_on_input_size_auth_and_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Malformed inputs, oversized bodies, missing auth, and bad JSON all fail.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors") + destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6" + with pytest.raises(ValueError, match="destination"): + module.assert_tag_absent( + "registry.invalid/x:tag", username="robot", password="private" + ) + with pytest.raises(RuntimeError, match="credentials"): + module.assert_tag_absent(destination, username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"x" * 1_048_577), + ) + with pytest.raises(RuntimeError, match="invalid artifact JSON"): + module.verify_registry_digest( + destination, + "sha256:" + "2" * 64, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"\xff"), + ) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + with pytest.raises(RuntimeError, match="unavailable"): + module._credentials() + + +def test_default_harbor_opener_returns_http_responses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real wrapper returns both normal and non-redirect HTTP responses.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_opener") + request = module.urllib.request.Request("https://registry.bstein.dev/test") + + class SuccessOpener: + def open(self, _request, timeout): + assert timeout == 7 + return Response(204) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener() + ) + assert module._registry_request(request, 7).status == 204 + + class ErrorOpener: + def open(self, _request, timeout): + assert timeout == 8 + raise module.urllib.error.HTTPError( + _request.full_url, 404, "missing", {}, None + ) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener() + ) + assert module._registry_request(request, 8).code == 404 + + +@pytest.mark.parametrize( + ("status", "artifact", "message"), + [ + (404, {}, "HTTP 404"), + (200, {}, "omitted"), + (200, {"digest": "sha256:" + "2" * 64}, "does not match"), + (200, {"digest": "latest"}, "omitted"), + ], +) +def test_registry_verification_rejects_missing_or_mismatched_digest( + status: int, artifact: dict[str, str], message: str +) -> None: + """Rendering requires an independent exact Harbor digest response.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry") + digest = "sha256:" + "1" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4" + body = json.dumps(artifact).encode("utf-8") + with pytest.raises(RuntimeError, match=message): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response(status, body=body), + ) + + +def test_registry_verification_accepts_exact_pushed_digest() -> None: + """An exact authenticated Harbor digest allows artifact rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok") + digest = "sha256:" + "4" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5" + with pytest.raises(RuntimeError, match="expected tag"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + {"digest": digest, "tags": [{"name": "different-tag"}]} + ).encode("utf-8"), + ), + ) + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": True, + } + ], + } + ).encode("utf-8"), + ), + ) + + +def test_registry_verification_rejects_tag_without_server_immutability() -> None: + """Digest evidence is insufficient unless Harbor reports the build tag immutable.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_mutable") + digest = "sha256:" + "7" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'8' * 40}-build-19" + captured = {} + + def opener(request, _timeout): + captured["url"] = request.full_url + return Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": False, + } + ], + } + ).encode(), + ) + + with pytest.raises(RuntimeError, match="immutable"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=opener, + ) + assert captured["url"].endswith("?with_immutable_status=true") + + +def test_release_main_preflight_uses_fixed_credentials_and_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The preflight CLI validates and checks exactly the requested build tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent") + revision = "3" * 40 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12" + captured = {} + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + + def assert_absent(value, **credentials): + captured["destination"] = value + captured["credentials"] = credentials + + def verify_policy(**credentials): + captured["policy_credentials"] = credentials + + monkeypatch.setattr(module, "assert_tag_absent", assert_absent) + monkeypatch.setattr(module, "verify_immutable_policy", verify_policy) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "assert-absent", + "--source-revision", + revision, + "--build-number", + "12", + "--destination", + destination, + ], + ) + assert module.main() == 0 + assert captured == { + "destination": destination, + "credentials": {"username": "robot", "password": "private"}, + "policy_credentials": {"username": "robot", "password": "private"}, + } + + +def test_renderer_skips_a_matching_entry_without_a_digest() -> None: + """Only the one complete image entry is changed when an earlier entry drifts.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan") + digest = "sha256:" + "5" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: ignored\n" + " - name: example.invalid/other\n" + " newTag: stable\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + ) + assert module.render_kustomization(source, digest).endswith(f"{digest}\n") + + +@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308]) +def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None: + """Generic proxy successes and redirects are not proof of a queued build.""" + module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + status, {"Location": "https://ci.bstein.dev/queue/item/9/"} + ) + with pytest.raises(RuntimeError, match=f"HTTP {status}"): + module.trigger_build("a" * 40, token_file=token, opener=response) + + +@pytest.mark.parametrize( + "location", + [ + "", + "https://evil.invalid/queue/item/9/", + "http://ci.bstein.dev/queue/item/9/", + "https://ci.bstein.dev/job/hermes-agent-image/9/", + "https://ci.bstein.dev/queue/item/9/?token=leak", + "https://ci.bstein.dev/queue/item/9/#fragment", + "https://ci.bstein.dev/queue/item/not-a-number/", + ], +) +def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location( + tmp_path: Path, location: str +) -> None: + """A real accepted status still needs the exact same-origin queue resource.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + 201, {"Location": location} + ) + with pytest.raises(RuntimeError, match="Location"): + module.trigger_build("b" * 40, token_file=token, opener=response) + + +def test_trigger_uses_https_and_accepts_exact_relative_queue_path( + tmp_path: Path, +) -> None: + """A relative plugin Location is resolved only against the fixed HTTPS origin.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + captured = {} + + def opener(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response(201, {"Location": "/queue/item/11/"}) + + result = module.trigger_build("c" * 40, token_file=token, opener=opener) + assert captured["url"].startswith("https://ci.bstein.dev/") + assert captured["timeout"] == 20 + assert result["queue_path"] == "/queue/item/11/" diff --git a/testing/tests/test_hermes_image_builder_coverage.py b/testing/tests/test_hermes_image_builder_coverage.py new file mode 100644 index 00000000..13a33177 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_coverage.py @@ -0,0 +1,456 @@ +"""Boundary coverage for the three security-critical image-lane helpers.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) +TRIGGER = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _robot(module, *, immutable: bool = False, duration=-1) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "duration": duration, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": module.PROJECT, + "access": access, + } + ], + } + + +def _robot_list(module, *, robot_id=41, name=None): + return json.dumps( + [{"id": robot_id, "name": name or module.PUBLISH_ROBOT}] + ).encode() + + +def test_runner_rejects_directive_and_heredoc_boundaries(tmp_path: Path) -> None: + """Malformed parser state, bodies, sizes, and indices all fail closed.""" + module = _load(RUNNER, "runner_boundary_coverage") + assert module._escape_character([""]) == "\\" + with pytest.raises(ValueError, match="multiple"): + module._escape_character(["# escape=\\", "# escape=\\"]) + with pytest.raises(ValueError, match="unsupported"): + module._escape_character(["# escape=^"]) + with pytest.raises(ValueError, match="unterminated.*continuation"): + module._logical_instruction(["R\\"], 0, "\\") + with pytest.raises(ValueError, match="unterminated node"): + module.extract_blocks("RUN node <<'NODE'\nbody") + + source = DOCKERFILE.read_text(encoding="utf-8") + marker = "RUN node <<'NODE'\n" + body_start = source.index(marker) + len(marker) + body_end = source.index("\nNODE", body_start) + empty = source[:body_start] + source[body_end + 1 :] + with pytest.raises(ValueError, match="empty node"): + module.extract_blocks(empty) + + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + with dockerfile.open("wb") as stream: + stream.truncate(module.MAX_DOCKERFILE_BYTES + 1) + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + dockerfile.write_text(source, encoding="utf-8") + for index in (0, 10): + with pytest.raises(ValueError, match="index"): + module.replay(dockerfile, index) + + +def test_runner_main_dispatches_the_exact_path_and_index( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The command-line wrapper cannot omit either bounded input.""" + module = _load(RUNNER, "runner_main_coverage") + dockerfile = tmp_path / "Dockerfile" + seen = [] + monkeypatch.setattr(module, "replay", lambda path, index: seen.append((path, index))) + monkeypatch.setattr( + sys, + "argv", + ["runner", "--dockerfile", str(dockerfile), "--block-index", "4"], + ) + assert module.main() == 0 + assert seen == [(dockerfile, 4)] + + +def test_trigger_redirect_wrapper_accepts_only_plugin_303( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The low-level opener returns a real response or the one expected redirect.""" + module = _load(TRIGGER, "trigger_opener_coverage") + assert module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None + request = module.urllib.request.Request(module.JENKINS_BUILD_URL) + + class Opener: + def __init__(self, result) -> None: + self.result = result + + def open(self, _request, timeout): + assert timeout == 9 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + success = Response(201) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(success) + ) + assert module._open_without_redirect(request, 9) is success + redirect = module.urllib.error.HTTPError( + request.full_url, 303, "queued", {"Location": "/queue/item/1/"}, None + ) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(redirect) + ) + assert module._open_without_redirect(request, 9) is redirect + denied = module.urllib.error.HTTPError(request.full_url, 403, "denied", {}, None) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(denied) + ) + with pytest.raises(urllib.error.HTTPError): + module._open_without_redirect(request, 9) + + +def test_trigger_main_reports_safe_success_and_errors( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """CLI output contains safe metadata and converts expected failures to JSON.""" + module = _load(TRIGGER, "trigger_main_coverage") + revision = "a" * 40 + monkeypatch.setattr(sys, "argv", ["trigger", revision]) + monkeypatch.setattr( + module, + "trigger_build", + lambda value: {"job": module.JOB_NAME, "source_revision": value, "status": 201}, + ) + assert module.main() == 0 + assert json.loads(capsys.readouterr().out)["source_revision"] == revision + monkeypatch.setattr( + module, "trigger_build", lambda _value: (_ for _ in ()).throw(OSError("closed")) + ) + assert module.main() == 1 + assert json.loads(capsys.readouterr().out) == {"error": "closed"} + + +def test_harbor_http_client_bounds_requests_and_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Credentials stay same-origin and all response/error paths remain bounded.""" + module = _load(HARBOR, "harbor_client_coverage") + assert module.NoRedirect().redirect_request(None, None, 0, None, None, None) is None + + class Opener: + result = Response(200, b"ok", {"X-Test": "yes"}) + + def open(self, request, timeout): + self.request = request + assert timeout == 20 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + opener = Opener() + monkeypatch.setattr(module.urllib.request, "build_opener", lambda *_args: opener) + client = module.HarborClient(module.EXPECTED_ORIGIN + "/", "admin", "private") + status, body, headers = client.request("POST", "/rules", {"value": 1}) + assert (status, body, headers["X-Test"]) == (200, b"ok", "yes") + assert opener.request.get_header("Content-type") == "application/json" + + opener.result = module.urllib.error.HTTPError( + module.EXPECTED_ORIGIN, 404, "missing", {}, io.BytesIO(b"missing") + ) + assert client.request("GET", "/missing")[:2] == (404, b"missing") + opener.result = module.urllib.error.URLError("offline") + with pytest.raises(module.HarborUnavailable): + client.request("GET", "/rules") + opener.result = Response(200, b"x" * (module.MAX_RESPONSE + 1)) + with pytest.raises(RuntimeError, match="size"): + client.request("GET", "/rules") + + +@pytest.mark.parametrize("body", [b"\xff", b"not-json", b"{}", b"[1]"]) +def test_harbor_json_lists_reject_invalid_shapes(body: bytes) -> None: + """Malformed encodings, JSON, objects, and scalar members are never lists.""" + module = _load(HARBOR, f"harbor_json_{body!r}") + with pytest.raises(RuntimeError, match="invalid"): + module._json_list(body, "fixture") + + +@pytest.mark.parametrize( + "robot,match", + [ + ({"permissions": "bad"}, "permissions"), + ({"permissions": [1]}, "permissions"), + ({"permissions": []}, "one existing"), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []}, + {"kind": "project", "namespace": "bstein", "access": []}, + ] + }, + "one existing", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": "bad"} + ] + }, + "access", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []} + ] + }, + "pull/push", + ), + ], +) +def test_publisher_scope_rejects_malformed_or_incomplete_permissions( + robot: dict, match: str +) -> None: + """Robot mutations require one structurally exact existing push scope.""" + module = _load(HARBOR, f"harbor_scope_{match}") + with pytest.raises(RuntimeError, match=match): + module._publisher_scope(robot) + assert module._verified_publisher(robot) is False + + +@pytest.mark.parametrize( + "responses,match,unavailable", + [ + ([(503, b"", {})], "503", True), + ([(403, b"", {})], "403", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b'[{\"name\":\"robot$jenkins-pipelines\",\"id\":0}]', _count(1))], "valid ID", False), + ], +) +def test_publisher_list_rejects_status_count_and_id( + responses, match: str, unavailable: bool +) -> None: + """Publisher discovery distinguishes readiness from permanent policy errors.""" + module = _load(HARBOR, f"harbor_publisher_list_{match}_{unavailable}") + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +@pytest.mark.parametrize( + "second,match,unavailable", + [ + ((503, b"", {}), "503", True), + ((403, b"", {}), "403", False), + ((200, b"not-json", {}), "invalid robot JSON", False), + ((200, b"[]", {}), "invalid shape", False), + ((200, b'{}', {}), "not active and exact", False), + ], +) +def test_publisher_read_rejects_status_and_identity( + second, match: str, unavailable: bool +) -> None: + """The detailed robot reread must be available, valid, and exact.""" + module = _load(HARBOR, f"harbor_publisher_read_{match}_{unavailable}") + responses = [(200, _robot_list(module), _count(1)), second] + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_update_rejects_duration_reference_and_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only the preserved scope and a successful policy PUT can be accepted.""" + module = _load(HARBOR, "harbor_publisher_update_coverage") + for duration in ("forever",): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module, duration=duration)).encode(), {}), + ] + with pytest.raises(RuntimeError, match="duration"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + robot = _robot(module) + monkeypatch.setattr(module, "_publisher_scope", lambda _robot: ({}, [])) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(robot).encode(), {}), + ] + with pytest.raises(RuntimeError, match="normalization"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + monkeypatch.undo() + for status, error in ((503, module.HarborUnavailable), (409, RuntimeError)): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (status, b"", {}), + ] + with pytest.raises(error): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_and_rule_verification_retries_are_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stale reads may retry, but never silently become successful evidence.""" + module = _load(HARBOR, "harbor_retry_coverage") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (200, b"", {}), + *((500, b"", {}) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + rule_path = "/api/v2.0/projects/bstein/immutabletagrules/23" + responses = [ + (200, b"[]", _count(0)), + (201, b"", {"Location": rule_path}), + *((200, b"[]", _count(0)) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_rule(FakeClient(responses)) + + +def test_rule_rejects_ids_create_status_and_location_suffix() -> None: + """Neither existing nor newly created rules can omit their exact positive ID.""" + module = _load(HARBOR, "harbor_rule_id_coverage") + bad_rule = {"id": 0, **module.EXPECTED_RULE} + with pytest.raises(RuntimeError, match="valid ID"): + module.ensure_rule( + FakeClient([(200, json.dumps([bad_rule]).encode(), _count(1))]) + ) + with pytest.raises(module.HarborUnavailable): + module.ensure_rule( + FakeClient([(200, b"[]", _count(0)), (503, b"", {})]) + ) + with pytest.raises(RuntimeError, match="invalid ID"): + module.ensure_rule( + FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + { + "location": "/api/v2.0/projects/bstein/immutabletagrules/nope" + }, + ), + ] + ) + ) + + +def test_harbor_main_retries_then_reports_exact_ids( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Runtime credentials are required and transient startup failures are bounded.""" + module = _load(HARBOR, "harbor_main_coverage") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + attempts = iter([module.HarborUnavailable("warming"), 17]) + + def ensure_rule(_client): + result = next(attempts) + if isinstance(result, BaseException): + raise result + return result + + monkeypatch.setattr(module, "ensure_rule", ensure_rule) + monkeypatch.setattr(module, "ensure_publisher_can_read_rule", lambda _client: 41) + monkeypatch.setattr(module.time, "sleep", lambda seconds: None) + assert module.main() == 0 + assert "id=17" in capsys.readouterr().out + + password_file.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.main() diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py new file mode 100644 index 00000000..27ccc9b3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -0,0 +1,335 @@ +"""Fresh fail-closed review cases for the Hermes image builder.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: + """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" + spec = _pod_spec() + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true", + } + affinity = spec["affinity"]["nodeAffinity"] + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [ + { + "weight": 100, + "preference": { + "matchExpressions": [ + {"key": "hardware", "operator": "In", "values": ["rpi5"]} + ] + }, + } + ] + expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][ + "nodeSelectorTerms" + ][0]["matchExpressions"] + host_rule = next( + rule for rule in expressions if rule["key"] == "kubernetes.io/hostname" + ) + assert host_rule["operator"] == "NotIn" + assert set(host_rule["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-22", + "titan-24", + } + + +@pytest.mark.parametrize( + "unsupported", + [ + "RUN cat < None: + """Any RUN heredoc outside the reviewed nine is rejected before replay.""" + module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: + """Current source inventories cleanly and enables replay only for Kaniko.""" + module = _load(RUNNER, "heredoc_positive_contract") + source = DOCKERFILE.read_text(encoding="utf-8") + assert len(module.extract_blocks(source)) == 9 + assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 + assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") + pipeline = PIPELINE.read_text(encoding="utf-8") + assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_appended_tenth_reviewed_form_rejects_before_execution( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Even an otherwise supported marker cannot expand the nine-block set.""" + module = _load(RUNNER, "heredoc_tenth_block") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="contract changed"): + module.replay(dockerfile, 1) + assert calls == [] + + +@pytest.mark.parametrize( + ("prefix", "unsupported"), + [ + ("", "R\\\nUN cat < None: + """Split RUN/opcode operators and backtick escapes cannot evade inventory.""" + module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_merged_main_replays_each_block_before_dependent_work() -> None: + """PR13's blocked-task regression runs only after its source patch replay.""" + source = DOCKERFILE.read_text(encoding="utf-8") + replay_positions = [ + source.index(f"--block-index {index}") for index in range(1, 10) + ] + # Locate repeated markers rather than trusting one synthetic occurrence. + block_positions = [] + for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"): + start = 0 + while True: + position = source.find(marker, start) + if position < 0: + break + block_positions.append(position) + start = position + len(marker) + block_positions.sort() + assert len(block_positions) == len(replay_positions) == 9 + for index, (block, replay) in enumerate(zip(block_positions, replay_positions)): + assert block < replay + if index + 1 < len(block_positions): + assert replay < block_positions[index + 1] + regression = source.index( + "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py" + ) + assert replay_positions[4] < regression < replay_positions[5] + + +def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None: + """The build Pod selects one tokenless SA that no tracked RBAC grants bind.""" + account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8")) + assert account == { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"}, + "automountServiceAccountToken": False, + } + kustomization = yaml.safe_load( + (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8") + ) + assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"] + + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + for volume in spec.get("volumes", []): + projected = volume.get("projected", {}) + assert all("serviceAccountToken" not in item for item in projected.get("sources", [])) + + for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"): + for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")): + if not isinstance(document, dict) or document.get("kind") not in { + "RoleBinding", + "ClusterRoleBinding", + }: + continue + subjects = document.get("subjects") or [] + assert not any( + item.get("kind") == "ServiceAccount" + and item.get("name") == "hermes-image-builder" + for item in subjects + ) + + +def _release_fixture(tmp_path: Path): + module = _load(RELEASE, f"release_evidence_{tmp_path.name}") + digest = "sha256:" + "7" * 64 + revision = "8" * 40 + build = "23" + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" + digest_file = tmp_path / "hermes-agent.digest" + image_file = tmp_path / "hermes-agent.image" + manifest = tmp_path / "kustomization.yaml" + output = tmp_path / "release" + digest_file.write_text(digest + "\n", encoding="utf-8") + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number=build, + destination=destination, + kustomization=manifest, + output_dir=output, + ) + kwargs = { + "digest_file": digest_file, + "image_file": image_file, + "source_revision": revision, + "build_number": build, + "destination": destination, + "kustomization": manifest, + "output_dir": output, + } + return module, kwargs + + +def test_success_evidence_revalidation_accepts_only_exact_complete_set( + tmp_path: Path, +) -> None: + """Exact evidence passes; an extra, missing, or altered file fails closed.""" + module, kwargs = _release_fixture(tmp_path) + module.validate_release_artifacts(**kwargs) + + extra = kwargs["output_dir"] / "unexpected" + extra.write_text("surprise\n", encoding="utf-8") + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + extra.unlink() + + metadata = kwargs["output_dir"] / "hermes-agent-image.json" + original = metadata.read_text(encoding="utf-8") + metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"')) + with pytest.raises(ValueError, match="incomplete or mismatched"): + module.validate_release_artifacts(**kwargs) + metadata.write_text(original, encoding="utf-8") + (kwargs["output_dir"] / "hermes-image-update.patch").unlink() + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + + +def test_verify_evidence_cli_needs_no_runtime_registry_credential( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Post-success validation is deterministic and cannot mask missing creds.""" + module, kwargs = _release_fixture(tmp_path) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_image_release.py", + "verify-evidence", + "--digest-file", + str(kwargs["digest_file"]), + "--image-file", + str(kwargs["image_file"]), + "--source-revision", + kwargs["source_revision"], + "--build-number", + kwargs["build_number"], + "--destination", + kwargs["destination"], + "--kustomization", + str(kwargs["kustomization"]), + "--output-dir", + str(kwargs["output_dir"]), + ], + ) + assert module.main() == 0 + + +def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None: + """Missing or partial post-success evidence must change the build to failed.""" + source = PIPELINE.read_text(encoding="utf-8") + post = source.split(" post {", 1)[1] + assert "success {" in post + assert "always {" not in post + assert "verify-evidence" in post + assert 'allowEmptyArchive: false' in post + archive = post.split("artifacts: '", 1)[1].split("'", 1)[0] + paths = archive.split(",") + assert len(paths) == 6 + assert len(set(paths)) == 6 + assert all("*" not in path for path in paths) + assert "find build -type f" in post diff --git a/testing/tests/test_hermes_image_builder_harbor.py b/testing/tests/test_hermes_image_builder_harbor.py new file mode 100644 index 00000000..9a64e24a --- /dev/null +++ b/testing/tests/test_hermes_image_builder_harbor.py @@ -0,0 +1,325 @@ +"""Server-side Harbor immutability contracts for Hermes agent releases.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("harbor_immutability", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rule(module, *, rule_id: int = 17, disabled: bool = False) -> dict: + return {"id": rule_id, **module.EXPECTED_RULE, "disabled": disabled} + + +def _robot(module, *, immutable: bool = False, extra_immutable: bool = False) -> dict: + access = [ + {"resource": "repository", "action": "pull", "effect": "allow"}, + {"resource": "repository", "action": "push", "effect": "allow"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + if extra_immutable: + access.append({"resource": "immutable-tag", "action": "delete"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "Jenkins publisher", + "level": "system", + "duration": -1, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": "other", + "access": [{"resource": "repository", "action": "pull"}], + }, + {"kind": "project", "namespace": "bstein", "access": access}, + ], + } + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +class FakeClient: + """Small deterministic Harbor API fake.""" + + def __init__(self, responses): + self.origin = "https://registry.bstein.dev/api/v2.0" + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def test_flux_tracks_exact_immutable_rule_before_jenkins() -> None: + """The Harbor rule is a reviewed prerequisite, not a Jenkins preflight only.""" + harbor = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml" + ).read_text() + ) + jenkins = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml" + ).read_text() + ) + check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "harbor-hermes-agent-immutability-ensure-1", + "namespace": "harbor", + } + assert check in harbor["spec"]["healthChecks"] + assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]} + + +def test_policy_job_uses_runtime_vault_secret_and_hardened_pinned_image() -> None: + """No Harbor credential is committed or retained in a mutable workload.""" + job = yaml.safe_load( + (REPO_ROOT / "services/harbor/hermes-agent-immutability-job.yaml").read_text() + ) + template = job["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap" + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"] + == "kv/data/atlas/harbor/harbor-core" + ) + pod = template["spec"] + assert pod["serviceAccountName"] == "harbor-policy-bootstrap" + assert pod["enableServiceLinks"] is False + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text() + policy = vault.split("harbor_policy_bootstrap_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/harbor/harbor-core"' in policy + assert 'capabilities = ["read"]' in policy + assert "*" not in policy + assert 'bound_service_account_names="harbor-policy-bootstrap"' in vault + script = SCRIPT.read_text() + assert 'PUBLISH_ROBOT = "robot$jenkins-pipelines"' in script + assert '{"resource": "immutable-tag", "action": "list"}' in script + assert 'client.request(\n "PUT", f"/robots/{robot_id}", payload' in script + assert '"secret"' not in script.split("payload = {", 1)[1].split("}", 1)[0] + + +def test_rule_contract_is_exact_repository_and_unique_build_tags() -> None: + """Unrelated bstein repositories and ordinary Hermes tags stay mutable.""" + module = _load_module() + assert module.EXPECTED_RULE == { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + + +def test_existing_exact_rule_is_idempotent_without_mutation() -> None: + """A rerun only validates the exact enabled rule.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + client = FakeClient([(200, body, _count(1))]) + assert module.ensure_rule(client) == 17 + assert [call[0] for call in client.calls] == ["GET"] + + +def test_create_requires_201_exact_location_and_verified_reread() -> None: + """Policy bootstrap fails closed until Harbor returns the exact persisted rule.""" + module = _load_module() + body = json.dumps([_rule(module, rule_id=23)]).encode() + client = FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + {"Location": "/api/v2.0/projects/bstein/immutabletagrules/23"}, + ), + (200, body, _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert client.calls[1] == ( + "POST", + "/projects/bstein/immutabletagrules", + module.EXPECTED_RULE, + ) + + +@pytest.mark.parametrize( + "responses,match", + [ + ([(200, b"[]", _count(0)), (200, b"", {})], "HTTP 200"), + ( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": "https://evil.invalid/1"}), + ], + "Location", + ), + ], +) +def test_create_rejects_noncanonical_responses(responses, match: str) -> None: + """Proxy success pages and foreign locations can never count as enforcement.""" + module = _load_module() + with pytest.raises(RuntimeError, match=match): + module.ensure_rule(FakeClient(responses)) + + +def test_readiness_failures_are_distinct_from_policy_rejections() -> None: + """The tracked Job may wait for Harbor without retrying an auth denial.""" + module = _load_module() + with pytest.raises(module.HarborUnavailable): + module.ensure_rule(FakeClient([(503, b"", {})])) + with pytest.raises(RuntimeError, match="HTTP 403") as exc: + module.ensure_rule(FakeClient([(403, b"", {})])) + assert not isinstance(exc.value, module.HarborUnavailable) + + +def test_disabled_or_duplicate_exact_scope_fails_closed() -> None: + """Bootstrap never silently edits a conflicting security policy.""" + module = _load_module() + disabled = json.dumps([_rule(module, disabled=True)]).encode() + with pytest.raises(RuntimeError, match="not enabled"): + module.ensure_rule(FakeClient([(200, disabled, _count(1))])) + + duplicate = json.dumps([_rule(module), _rule(module, rule_id=18)]).encode() + with pytest.raises(RuntimeError, match="multiple"): + module.ensure_rule(FakeClient([(200, duplicate, _count(2))])) + + +def test_policy_lists_reject_missing_or_truncated_count_evidence() -> None: + """A hidden second page can never produce a duplicate rule or robot update.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + for headers in ({}, _count(2)): + with pytest.raises(RuntimeError, match="count|truncated"): + module.ensure_rule(FakeClient([(200, body, headers)])) + robot_body = json.dumps( + [{"id": 41, "name": module.PUBLISH_ROBOT}] + ).encode() + with pytest.raises(RuntimeError, match="truncated"): + module.ensure_publisher_can_read_rule( + FakeClient([(200, robot_body, _count(2))]) + ) + + +def test_publisher_policy_adds_only_read_access_and_preserves_robot() -> None: + """Bootstrap preserves every existing scope and never touches robot secret state.""" + module = _load_module() + original = _robot(module) + persisted = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(original).encode(), {}), + (200, b"", {}), + (200, json.dumps(persisted).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "secret" not in payload + assert payload["permissions"][0] == original["permissions"][0] + assert payload["permissions"][1]["access"][-1] == { + "resource": "immutable-tag", + "action": "list", + } + + +def test_publisher_policy_is_idempotent_and_rejects_broad_access() -> None: + """An exact read policy is stable; mutation-capable immutable access is blocked.""" + module = _load_module() + exact = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(exact).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert [call[0] for call in client.calls] == ["GET", "GET"] + + broad = _robot(module, immutable=True, extra_immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(broad).encode(), {}), + ] + ) + with pytest.raises(RuntimeError, match="broader"): + module.ensure_publisher_can_read_rule(client) + + +def test_harbor_client_rejects_plaintext_or_ambiguous_origins() -> None: + """Runtime admin credentials are never sent over HTTP or a URL with query state.""" + module = _load_module() + for origin in ( + "http://registry.bstein.dev/api/v2.0", + "https://registry.bstein.dev/api/v2.0?next=evil", + "https://other.invalid/api/v2.0", + "", + ): + with pytest.raises(ValueError): + module.HarborClient(origin, "admin", "secret") diff --git a/testing/tests/test_hermes_image_builder_vault.py b/testing/tests/test_hermes_image_builder_vault.py new file mode 100644 index 00000000..f46823f5 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_vault.py @@ -0,0 +1,329 @@ +"""Vault least-privilege and fail-closed seeding tests for image releases.""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" +SEEDER = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" +) + + +def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None: + """Only the Jenkins controller and Hermes agent can read the fixed-job token.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source + assert ( + '"jenkins/* shared/harbor-pull quality/sonarqube-oidc ' + 'hermes/developer-jenkins" ""' + ) in source + assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source + assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source + assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source + assert '"hermes/chat-telegram" ""' in source + assert '"hermes/developer-jenkins"' not in source + + jenkins_spc = yaml.safe_load( + (REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text() + ) + assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync" + switchyard = yaml.safe_load( + (REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text() + ) + annotations = switchyard["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard" + + +def test_flux_orders_seed_then_jenkins_then_hermes() -> None: + """The fixed token must be Ready before either consumer is rolled.""" + app_root = REPO_ROOT / "clusters/atlas/flux-system/applications" + vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text()) + seed = yaml.safe_load( + (app_root / "vault-hermes-jenkins-token-seed/kustomization.yaml").read_text() + ) + jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text()) + hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text()) + role_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-k8s-auth-hermes-9", + "namespace": "vault", + } + seed_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-hermes-jenkins-build-token-seed-2", + "namespace": "vault", + } + assert role_check in vault["spec"]["healthChecks"] + assert {item["name"] for item in seed["spec"]["dependsOn"]} == {"vault"} + assert seed_check in seed["spec"]["healthChecks"] + assert jenkins["spec"]["suspend"] is False + assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= { + "helm", + "harbor", + "vault-hermes-jenkins-token-seed", + } + assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]} + + +def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None: + """The prerequisite is a tracked one-shot on healthy non-reserved capacity.""" + job = yaml.safe_load( + ( + REPO_ROOT / "services/vault-hermes-jenkins-token-seed/job.yaml" + ).read_text() + ) + assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-2" + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-jenkins-token-seed" + assert pod["enableServiceLinks"] is False + assert pod["restartPolicy"] == "Never" + assert pod["nodeSelector"]["hardware"] == "rpi5" + expression = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert set(expression["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-24", + } + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + source = SEEDER.read_text(encoding="utf-8") + assert '"options":{"cas":0}' in source + assert 'vault kv get -field=build_token "${secret_path}"' in source + assert "kv/data/atlas/hermes/developer-jenkins" in source + assert '"build_token"[[:space:]]' not in source + assert "kv patch" not in source + assert "kv delete" not in source + assert SEEDER.name not in VAULT_CONFIG.read_text(encoding="utf-8") + + +def test_seed_vault_policy_is_exact_create_read_and_rng_only() -> None: + """The seed identity cannot rotate, delete, list, or touch unrelated KV.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + policy = source.split("hermes_jenkins_token_seed_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/hermes/developer-jenkins"' in policy + assert 'capabilities = ["create", "read"]' in policy + assert 'path "sys/tools/random/32"' in policy + assert 'capabilities = ["update"]' in policy + for forbidden in ("delete", "patch", "list", 'kv/data/atlas/hermes/*'): + assert forbidden not in policy + assert 'bound_service_account_names="hermes-jenkins-token-seed"' in source + assert 'bound_service_account_namespaces="vault"' in source + + +def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]: + """Create deterministic Vault/sleep fakes and return their evidence paths.""" + binary_dir = tmp_path / "bin" + binary_dir.mkdir(parents=True) + calls = tmp_path / "calls" + stdin_capture = tmp_path / "stdin" + vault = binary_dir / "vault" + vault.write_text( + """#!/bin/sh +set -eu +printf '%s\n' "$*" >> "$FAKE_CALL_LOG" +scenario="$FAKE_SCENARIO" +if [ "$1" = "write" ] && [ "$2" = "-field=random_bytes" ] && [ "$3" = "sys/tools/random/32" ]; then + printf '%064d\n' 0 + exit 0 +fi +if [ "$1" = "kv" ] && [ "$2" = "get" ] && [ "$3" = "-field=build_token" ]; then + case "$scenario" in + existing) + printf '%064d\n' 1 + exit 0 + ;; + cas-race) + printf '%064d\n' 2 + exit 0 + ;; + field-prefix) + printf 'prefix%064d\n' 3 + exit 0 + ;; + field-suffix) + printf '%064dsuffix\n' 4 + exit 0 + ;; + field-multiline) + printf '%064d\nextra\n' 5 + exit 0 + ;; + missing-field|metadata-only|custom-metadata) + echo 'No value found at build_token' >&2 + exit 2 + ;; + esac +fi +if [ "$1" = "write" ] && [ "$2" = "kv/data/atlas/hermes/developer-jenkins" ]; then + payload="${3#@}" + cp "$payload" "$FAKE_STDIN_CAPTURE" + if [ "$scenario" = "cas-race" ]; then + : > "$FAKE_WINNER" + echo 'check-and-set parameter did not match' >&2 + exit 2 + fi + exit 0 +fi +if [ "$1" = "read" ] && [ "$2" = "-format=json" ]; then + case "$scenario" in + existing) + printf '{"data":{"data":{"build_token":"%064d"},"metadata":{"build_token":"decoy"}}}\n' 1 + exit 0 + ;; + missing-field) + printf '{"data":{"data":{"other":"preserve-me"}}}\n' + exit 0 + ;; + metadata-only) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"build_token":"%064d"}}}\n' 6 + exit 0 + ;; + custom-metadata) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"custom_metadata":{"build_token":"%064d"}}}}\n' 7 + exit 0 + ;; + field-prefix|field-suffix|field-multiline) + printf '{"data":{"data":{"build_token":"present"}}}\n' + exit 0 + ;; + absent) + echo 'Code: 404' >&2 + exit 2 + ;; + cas-race) + if [ -f "$FAKE_WINNER" ]; then + printf '{"data":{"data":{"build_token":"%064d"}}}\n' 2 + exit 0 + fi + echo 'Code: 404' >&2 + exit 2 + ;; + transient) + count=0 + if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi + count=$((count + 1)) + printf '%s' "$count" > "$FAKE_COUNT" + if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi + echo 'Code: 404' >&2 + exit 2 + ;; + read-error) + echo 'permission denied' >&2 + exit 2 + ;; + esac +fi +echo "unexpected fake Vault call: $*" >&2 +exit 99 +""", + encoding="utf-8", + ) + sleep = binary_dir / "sleep" + sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for executable in (vault, sleep): + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + return binary_dir, calls, stdin_capture + + +def _run_seeder( + tmp_path: Path, scenario: str +) -> tuple[subprocess.CompletedProcess, str]: + binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path) + env = { + **os.environ, + "PATH": f"{binary_dir}:{os.environ['PATH']}", + "VAULT_TOKEN": "test-token", + "FAKE_SCENARIO": scenario, + "FAKE_CALL_LOG": str(calls), + "FAKE_STDIN_CAPTURE": str(stdin_capture), + "FAKE_WINNER": str(tmp_path / "winner"), + "FAKE_COUNT": str(tmp_path / "count"), + "TMPDIR": str(tmp_path), + } + result = subprocess.run( + ["sh", str(SEEDER)], + env=env, + text=True, + capture_output=True, + check=False, + ) + call_text = calls.read_text(encoding="utf-8") if calls.exists() else "" + return result, call_text + + +@pytest.mark.parametrize( + "scenario", + ["existing", "missing-field", "metadata-only", "custom-metadata", "read-error"], +) +def test_seeder_never_overwrites_existing_or_ambiguous_state( + tmp_path: Path, scenario: str +) -> None: + """Existing fields and persistent read failures can never become a put.""" + result, calls = _run_seeder(tmp_path, scenario) + assert (result.returncode == 0) is (scenario == "existing") + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +@pytest.mark.parametrize("scenario", ["field-prefix", "field-suffix", "field-multiline"]) +def test_seeder_requires_the_entire_exact_data_field( + tmp_path: Path, scenario: str +) -> None: + """Prefix, suffix, or multiline values cannot satisfy readiness.""" + result, calls = _run_seeder(tmp_path, scenario) + assert result.returncode != 0 + assert "kv get -field=build_token kv/atlas/hermes/developer-jenkins" in calls + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret( + tmp_path: Path, +) -> None: + """A confirmed 404 produces one create-only write without a token argument.""" + result, calls = _run_seeder(tmp_path, "absent") + assert result.returncode == 0, result.stderr + assert calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert "@" in calls + assert "0000000000000000000000000000000000000000000000000000000000000000" not in calls + payload = (tmp_path / "stdin").read_text(encoding="utf-8") + assert '"options":{"cas":0}' in payload + assert '"build_token":"' in payload + assert not (tmp_path / "hermes-jenkins-token.json").exists() + + +def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner( + tmp_path: Path, +) -> None: + """Transient reads retry; a competing create is accepted only after reread.""" + transient, transient_calls = _run_seeder(tmp_path / "transient", "transient") + assert transient.returncode == 0, transient.stderr + assert transient_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 3 + assert transient_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + + race, race_calls = _run_seeder(tmp_path / "race", "cas-race") + assert race.returncode == 0, race.stderr + assert race_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert race_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 2 + assert "another seeder won CAS" in race.stderr diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..fa4c8021 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp "chat-relay-key": "relay-key", "gitea-token": "gitea-key", "gitea-username": "hermes-automation", + "jenkins-image-build-token": "job-scoped-token", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", "node-ssh-known-hosts": "known-hosts", @@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600 assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600 assert (runtime / "claude/settings.json").is_symlink() assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists() From 9d5214f809cfac3d092c8df687a9606d4c12976d Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 13:11:34 +0000 Subject: [PATCH 14/46] fix(hermes): close full-handoff acceptance blockers --- ci/scripts/semgrep_report.py | 47 +- docs/hermes_full_handoff_acceptance.md | 480 +++++++--------- scripts/ops/hermes_handoff_acceptance.py | 217 ++++++-- scripts/ops/hermes_handoff_catalog.py | 102 ++-- scripts/ops/hermes_handoff_checks_access.py | 303 ++++++---- scripts/ops/hermes_handoff_checks_delivery.py | 295 +++++----- scripts/ops/hermes_handoff_checks_platform.py | 78 ++- scripts/ops/hermes_handoff_checks_workers.py | 253 ++++----- scripts/ops/hermes_handoff_ephemeral.py | 524 ++++++++++++------ scripts/ops/hermes_handoff_evaluators.py | 267 ++++++++- scripts/ops/hermes_handoff_exec.py | 497 ++++++++++++----- scripts/ops/hermes_handoff_json_rules.py | 273 ++++++--- scripts/ops/hermes_handoff_model.py | 58 +- scripts/ops/hermes_handoff_policy.py | 433 +++++++++------ scripts/ops/hermes_handoff_redaction.py | 90 +-- scripts/ops/hermes_handoff_rules.py | 76 ++- scripts/ops/hermes_handoff_run.py | 217 ++++++-- testing/quality_contract.json | 89 +++ testing/quality_coverage.py | 56 +- testing/quality_handoff_mutation.py | 209 +++++++ testing/quality_hygiene.py | 25 +- testing/tests/conftest.py | 20 + testing/tests/test_hermes_auto_router.py | 6 + .../tests/test_hermes_handoff_acceptance.py | 171 ++++-- .../test_hermes_handoff_acceptance_armed.py | 120 ++++ testing/tests/test_hermes_handoff_catalog.py | 87 +-- .../tests/test_hermes_handoff_ephemeral.py | 510 +++++++++-------- .../tests/test_hermes_handoff_evaluators.py | 300 +++++++++- testing/tests/test_hermes_handoff_exec.py | 232 +++++++- .../tests/test_hermes_handoff_json_rules.py | 198 +++++-- testing/tests/test_hermes_handoff_model.py | 44 +- .../test_hermes_handoff_mutation_gate.py | 51 ++ testing/tests/test_hermes_handoff_policy.py | 276 +++++++-- .../tests/test_hermes_handoff_redaction.py | 44 +- testing/tests/test_hermes_handoff_rules.py | 47 +- testing/tests/test_hermes_handoff_run.py | 207 ++++++- testing/tests/test_hermes_handoff_support.py | 8 +- testing/tests/test_quality_contract.py | 64 ++- .../tests/test_quality_coverage_helpers.py | 69 ++- testing/tests/test_semgrep_report.py | 27 +- 40 files changed, 5121 insertions(+), 1949 deletions(-) create mode 100644 testing/quality_handoff_mutation.py create mode 100644 testing/tests/conftest.py create mode 100644 testing/tests/test_hermes_handoff_acceptance_armed.py create mode 100644 testing/tests/test_hermes_handoff_mutation_gate.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..f46e528d 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -27,7 +27,10 @@ def _read_json(path: Path) -> dict[str, Any]: except json.JSONDecodeError as exc: return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []} if not isinstance(payload, dict): - return {"errors": [{"message": "report payload is not an object"}], "results": []} + return { + "errors": [{"message": "report payload is not an object"}], + "results": [], + } return payload @@ -49,10 +52,18 @@ def build_report( blocking = blocking_severities or BLOCKING_SEVERITIES raw_results = semgrep_payload.get("results", []) raw_errors = semgrep_payload.get("errors", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + errors = ( + [item for item in raw_errors if isinstance(item, dict)] + if isinstance(raw_errors, list) + else [] + ) - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) @@ -93,16 +104,18 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) - end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line + end_line = _line_number( + end.get("line") if isinstance(end, dict) else None, start_line + ) return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL", "severity": SONAR_SEVERITY_BY_SEMGREP[severity], "primaryLocation": { - "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"), + "message": str( + extra.get("message") or finding.get("check_id") or "Semgrep finding" + ), "filePath": path, "textRange": { "startLine": start_line, @@ -112,11 +125,19 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: } -def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: +def build_sonar_issues( + semgrep_payload: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: """Build SonarQube generic external issues from Semgrep JSON output.""" raw_results = semgrep_payload.get("results", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + issues = [ + issue for finding in results if (issue := _sonar_issue_from_finding(finding)) + ] return {"issues": issues} @@ -144,7 +165,9 @@ def main(argv: list[str] | None = None) -> int: ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) if args.sonar_issues_output: sonar_output = Path(args.sonar_issues_output) sonar_output.parent.mkdir(parents=True, exist_ok=True) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index 8b9103c2..d7aa9608 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -1,323 +1,259 @@ -# Hermes full-handoff acceptance harness and release runbook +# Hermes full-handoff acceptance and release runbook -`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform -handoff is fit to release. It is read-only by default, it classifies every check -`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or -`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up -to evidence of absence: a probe that could not run, a catalog entry whose -evidence no longer exists, an expired deadline, or a harness bug all land in -`NOT_RUN` and stop the release. +`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It +contains 63 checks in 15 groups. An empty catalog, an unknown result status, a +mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the +decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the +default mode and are replaced with mandatory results only in armed mode. -``` +This runbook describes evidence collection. It does not authorize a merge, +publish, deployment, Flux reconcile, credential read, or live cluster mutation. +Human review of PR #19 and of the resulting report is required. + +## Required release inputs + +Every value that can go stale must be supplied explicitly. Resolve `main`, PR +heads, the image digest, build source SHA, deployment revision, and chat config +revision from their authoritative read-only sources immediately before the run. +Do not copy the snapshot in the dependency section into a later run. + +```sh scripts/ops/hermes_handoff_acceptance.py \ --context atlas-operator \ - --node-count 21 \ - --expected-suspension flux-system/bstein-dev-home-migrations \ + --remote-main-sha "$REMOTE_MAIN_SHA" \ + --reviewed-head-sha "$REVIEWED_PR19_HEAD" \ + --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \ + --build-sha "$IMAGE_BUILD_SOURCE_SHA" \ + --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \ + --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \ + --dependency-head "12=$PR12_HEAD" \ + --dependency-head "14=$PR14_HEAD" \ + --dependency-head "15=$PR15_HEAD" \ + --dependency-head "16=$PR16_HEAD" \ + --dependency-head "17=$PR17_HEAD" \ + --dependency-head "18=$PR18_HEAD" \ + --dependency-head "20=$PR20_HEAD" \ --output build/hermes-handoff-acceptance.json ``` -Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to -`--output` (or stdout); a short human summary goes to stderr unless -`--json-only` is passed. +The fixed topology is three pool workers on three distinct nodes. `--node-count` +and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment, +Telegram session continuity, and the chat revision are mandatory. Deliberate Flux +suspensions must be named with repeatable `--expected-suspension` arguments. -## Why two vantages +Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution +bounds. JSON is written to `--output` or stdout; the human summary goes to stderr +unless `--json-only` is passed. -Every authority claim is checked from two places that cannot cover for each -other: +## Structural safety boundary -| Vantage | What it is | What it proves | -| --- | --- | --- | -| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | -| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | -| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | +The catalog is validated before a `Runner` exists. Invalid inputs, an empty or +unsafe catalog, and an invalid armed preflight therefore create no subprocess. +The command policy is checked both before and after executable resolution. -Two rules keep this honest: +Default mode permits a narrow read grammar: -* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and - `--as-uid` are rejected by policy for any operator-side command. They are - reachable only as the *inner* command of a `kubectl exec`, where the identity - is testing its own refusal to impersonate rather than borrowing authority. - `kubectl auth can-i --as` is therefore not available as a substitute for a - real self-probe. -* **The two vantages must be different principals.** Before any check runs, the - harness asks each vantage `kubectl auth whoami` and compares. If the operator - and the in-pod probe authenticate as the same principal, the dual-vantage - evidence is a restatement rather than a corroboration, and the run is `NO_GO` - with `the operator and in-pod vantages authenticate as the same principal`. - Run the harness from a genuinely external kubeconfig — not from inside the - Hermes owner pod. +- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources, + raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing + projections, `auth reconcile`, config operations, all mutations, and every + dry-run mutation trick are rejected before spawn. +- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, + external diff helpers, and push are rejected. Helm values are not readable. +- Shell commands must be rendered from frozen templates, use fixed executable + paths, and cannot name service-account, Vault, runtime-access, SSH, or other + credential roots. +- Outer impersonation is rejected. A reviewed inner command can use impersonation + only after an explicit `kubectl exec --` boundary; it cannot substitute for the + independent self identity. +- `PATH` is replaced with a fixed path. Every outer executable is resolved to an + allowed real path, checked for unsafe write mode, and compared with a pinned + SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate + evidence. -Deny checks additionally require a **real refused request**, not only an -authorization review. `kubectl auth can-i` reports what RBAC says; only an -attempt proves the API server enforces it. Mutating attempts go through -`--dry-run=server`: the API server authorises the request and then discards it, -so a refusal is genuine and nothing is written either way. +The executor uses a new process group, drains stdout and stderr concurrently, +and retains at most the configured bytes while the child is running. A monotonic +absolute timeout terminates and reaps the group, including descendants that keep +pipes open. Timeout, deadline, output, and concurrency values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. +Truncation discards the affected stream and makes every evaluator `NOT_RUN`. -## Safety properties +Credential screening runs before truncation. Command, output, error, identity, +reason, evidence, JSON, and human-summary fields are screened recursively, +including mapping keys and short/numeric sensitive values. The complete rendered +report is screened again; a remaining credential shape is replaced by a bounded +`NO_GO` stub rather than being written. No credential prefix is retained at a +truncation boundary. -These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is -a policy error before a run rather than a mutation or a leak during one. +## Vantages and evidence -* **No mutation in the default mode.** Only read subcommands are reachable; - `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run - flag. `git push` and every non-`GET` forge method are refused outside the - armed mode. There is deliberately no live TokenRequest probe — a successful - one would mint a real credential. -* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, - `curl`, and `wget` are not on the binary allowlist. Secrets may only be read - with `-o name`. Environment probes list variable *names* and never values. - Provider health files are read through a frozen template that projects an - allow-listed set of non-secret status fields. The node account's lock state is - reported without reading its password hash; its authorized keys are counted, - never printed. -* **Shell is frozen.** `sh -c` is reachable only through the reviewed, - parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain - shell metacharacters. -* **Output is bounded and screened.** Each capture is truncated to a byte budget - *before* it is screened, so a large dump cannot outrun the screen. Bulk - evidence (the routing-log tail, the Kanban listing) is parsed but replaced with - `[output not recorded]` in the artifact. The screen preserves JSON quoting, so - screened structured output is still parsable. -* **The report is re-checked before it is published.** `unscreened_fields` walks - the rendered payload; if anything still looks credential-shaped the report is - replaced with a `NO_GO` stub naming the offending pointers, and the file that - would have leaked is never written. +The operator must use an external read-only kubeconfig. The self probe runs in +the actual Hermes agent pod. Both identities are obtained with +`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required +Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is +derived from all daemonset pod node names, not one selected pod. -## What the catalog asserts +Dangerous Kubernetes permissions are checked only with side-effect-free +`kubectl auth can-i` reviews. The default catalog never constructs a Secret, +TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one +forge administrative denial check performs a read-only GET and accepts only an +explicit authorization refusal; generic `404`, `not found`, and `no route` text +are not denial evidence. -The catalog is declarative (`hermes_handoff_catalog.py` plus the four -`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not -how it is plumbed. 71 checks in 14 groups: +The strict evaluators reject duplicate-key JSON, raw control characters, +`NaN`/infinity, boolean/number equality coercion, future timestamps, stale +provider evidence, and every malformed routing or Flux row. Routing freshness is +required separately for each provider, lane, and effort. Flux objects must be +non-suspended, current at `observedGeneration`, and explicitly `Ready=True`; +unknown or reconciling state is not healthy. -| Group | Assertions | -| --- | --- | -| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | -| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | -| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | -| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | -| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | -| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | -| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | -| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | -| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | -| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | -| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | -| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | -| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | -| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | +`release.exact-lineage-is-running` atomically binds: -### Site configuration +- fixed remote `main` SHA and local `origin/main`; +- open draft PR #19, base `main`, existing feature ref, and exact reviewed head; +- the exact `:git--build-@sha256:` image identity, + with the source SHA equal to the release `main` SHA; +- current Deployment generation and revision; +- the one active ReplicaSet revision/hash; every Ready pod image and imageID; +- the Flux-applied `main` revision. -A few checks are emitted only when the operator declares what to expect, -because a `names_present` assertion over an empty expectation would pass on any -workload at all — which reads as coverage without being any: +The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three +distinct nodes and claims, with service-account token automount disabled. The +chat checks always require durable Telegram sessions and the exact running chat +configuration revision. -| Flag | Enables | -| --- | --- | -| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | -| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | -| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | +## Armed ephemeral mode -`--node-count` must be the real node count; the default of `1` makes the -fleet-coverage check vacuous. +Armed mode is the only write path and runs only after the complete default +catalog reports `GO`. -### Catalog drift is a NO_GO, not a pass - -If a workload, field, or manifest is renamed, the affected checks report -`NOT_RUN` because the evidence they name is no longer there. That is the -intended behaviour: the release stops until either the platform or the catalog -is corrected. Do not "fix" a `NOT_RUN` by deleting the check. - -## Ephemeral mutation mode - -Proving that a worker can still open a draft pull request means actually opening -one. That is the only part of the harness that writes anything, and it is a -separate mode: - -``` +```sh scripts/ops/hermes_handoff_acceptance.py \ - --context atlas-operator \ + \ --arm-ephemeral-push \ --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ --ephemeral-token acceptance-20260817a \ --output build/hermes-handoff-acceptance-armed.json ``` -* The confirmation phrase must match exactly. -* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be - unique per run; the ref is always - `ephemeral/hermes-handoff-acceptance/`. -* **Preflight runs before any network call**: wrong phrase, wrong repository, - malformed token, or a protected push target all abort with `arming refused` - and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, - `production`, `stable`, `trunk`, `prod`, and `default` are refused by name - (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral - acceptance ref is refused by pattern. -* The guard is **exercised on every armed run** by - `ephemeral.protected-branch-refusal`, rather than trusting that it was checked - once. -* The push target is guarded; the pull-request *base* is not. Opening a draft - pull request against `main` writes nothing to `main`. The draft title always - starts with `WIP: `. -* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to - confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed - branch fails the run rather than passing it. Cleanup runs even when an earlier - step failed. -* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` - are refused even in armed mode. +Before any runner, network request, or cluster vantage is created, preflight +binds the working directory to the existing linked PR #19 worktree, the exact +feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base +`main`, confirmation phrase, and one unique ephemeral ref. The command policy is +then pinned to that exact ref. It allows only its push, one exact draft-create +payload, cleanup of the exactly discovered PR number, and deletion of that ref. -A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so -a read-only sweep can still reach `GO`. +The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the +pushed SHA, always discovers the PR after create (including ambiguous create +responses), and requires exactly one open draft with the expected base, head ref, +base SHA, and head SHA. Cleanup requires successful PR close, successful branch deletion, +a successful empty `ls-remote`, and a strict re-read of the same closed PR. +Malformed or partial cleanup is `FAIL`; it is never inferred from an error. -## Release order +Force, force-with-lease, fan-out, protected refs, other repositories, other PR +numbers, arbitrary PATCH payloads, and repository administration remain denied. -### Baseline, already merged and deployed +## Dependency, merge, and conflict graph -PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as -`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is -present; `baseline.origin-main-descends-merged-work` asserts it. Override with -`--baseline-commit` only when a later merge legitimately replaces it. +Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against +remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: -### Pending work, observed 2026-08-17T10:07Z - -These heads move. The harness re-fetches them at run time via -`baseline.pending-pull-requests-recorded`; the table is the snapshot this -document was written against, not an input. - -| PR | Branch | Head at observation | State | Delivers | +| PR | Head SHA | Draft | Base | Role | | --- | --- | --- | --- | --- | -| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | -| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | -| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | -| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | -| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | +| #12 | `465d386da5237f011cb408f1a3d4b7d0c6d8ffae` | no | `main` | AI usage exporter repair | +| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | +| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | +| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | +| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | +| #18 | `20002527512235b29054f3636b02418168373009` | yes | `main` | distributed three-worker pool | +| #19 | `8f005458282269ba5c07941814e4237f5d4cf3ac` | yes | `main` | reviewed blocked head; this repair replaces it on the same branch | +| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | -Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. +The repaired #19 SHA cannot be embedded in its own commit without changing that +SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; +the PR handoff records the exact value. All eight snapshot heads merge cleanly +with the snapshot `main` individually. Pairwise `git merge-tree --write-tree` +found conflicts only for: #12+#17, #14+#15, #14+#16, #14+#17, #14+#20, +#15+#16, #15+#17, #16+#20, and #17+#20. Gitea's `mergeable=false` on a draft is +not treated as conflict evidence. -### Merge order and why +Use this dependency order, refreshing every head and re-running merge-tree at +each step: -1. **#14 first.** It is the only change that *reduces* authority: it deletes - `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces - it with the namespaced/cluster observer roles, and it hardens the node - account. Merging it last would mean every earlier acceptance run measured a - cluster-admin agent. Every `access-denied` and `nodes` check depends on it. -2. **#16 second.** The image lane must exist before anything that needs a new - agent image is rolled out. It also touches `testing/quality_contract.json`, - so merging it early keeps later rebases small. `build.*` depends on it. -3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 - also touches — take #16's shape and rebase #15 onto it. `reliability.*` - depends on it. -4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` - and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 - also touch. Merging it after them avoids resolving the same conflict twice. -5. **#18 last.** The pool depends on the bounded forge client from #14, the - execution-safety patch from #15, and the agent image from #16. `pool.*` - depends on it. +1. #14 first so the release never restores or prolongs cluster-admin authority. +2. #16, then #15, resolving their Dockerfile and quality-contract changes + additively. Never take one PR's quality contract wholesale. +3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across + the current #14/#16 contract and helper changes. +4. #12 before #17, resolving their Deployment overlap once. +5. #17, then #18, after the authority, image, reliability, and reaping work. +6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are + merged and their exact supplied heads are ancestors of refreshed `main`. -Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` -(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), -`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), -`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), -`dockerfiles/Dockerfile.hermes-agent` (#15, #16), -`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), -`testing/tests/test_hermes_runtime_access.py` (#16, #18). +#19 overlaps `testing/quality_contract.json` with #14/#15/#16, +`testing/quality_coverage.py` and its helper tests with #14, +`test_hermes_auto_router.py` with #18, and existing Kustomize content with +#12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, +and all desired-state changes during conflict resolution. -PR #12 (AI usage exporter) is independent and can merge at any point; it touches -`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. +The #16/#19 integration must keep both the immutable digest and the source tag +(`newTag: git--build-` plus the digest) in rendered image identity. +Digest-only rendering loses the attested build-source link and therefore +correctly remains `NO_GO` under the exact-lineage evaluator. -### Image build and Flux rollout +## Safe rollback guidance -1. Merge in the order above; observe the default-branch Jenkins build to a - terminal result after each merge - (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). -2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an - immutable `git--build-` tag and emits - `build/hermes-agent-release/hermes-image-update.patch`. -3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, - and merge. The digest is what pins the deployment; - `build.deployed-image-is-digest-pinned` asserts it. -4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` - and `hermes-chat` are `Ready`, then wait for the rollout to settle. -5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an - in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled - deployment will legitimately fail the `identity`, `routing`, and `surfaces` - groups. +Every rollback is a reviewed forward fix or selective Git revert followed by +the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when +reverting unrelated portions of #14. If a rollback would widen RBAC, stop and +write a forward least-privilege fix instead. -### Rollback points +Reverting #18 removes the StatefulSet desired state, but its +`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no +delete retention policy is set. Do not claim Flux pruning deletes those PVCs. +Any PVC deletion is a separate, explicit, human-reviewed data-destruction task +with exact targets and recovery implications. -| After | Roll back by | Restores | -| --- | --- | --- | -| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | -| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | -| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | -| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | -| #17 merge | Revert the merge | Previous lane concurrency and node placement | -| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | +Image rollback means a reviewed commit restoring a previously attested digest; +never retag an immutable image. Do not use direct `kubectl` edits as a durable +rollback. -Every rollback is a Git revert reconciled by Flux. Do not roll back by direct -`kubectl` edit; the next reconcile would undo it. +## Quality and release evidence -## Go / no-go checklist +The canonical quality contract explicitly manages, lints, and tracks all 15 +`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and +branch coverage per tracked file and the 500-physical-line source cap. Contract +tests derive the module set so a future file cannot silently escape the gate. +Four pre-existing over-cap test files are temporarily locked to their exact +line count and SHA-256; any byte edit invalidates the exception and must split +the file below 500 lines. PR #14 supplies the intended additive test splits. -Run in order. Stop at the first item that does not hold. - -1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. -2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each - with a green default-branch build. -3. The agent image has been built by the #16 lane, is pinned by digest in - `services/hermes/kustomization.yaml`, and Flux has finished reconciling. -4. The harness is run from an **external operator kubeconfig**, not from inside - the Hermes owner pod, with `--node-count` set to the real node count and every - deliberate Flux suspension declared via `--expected-suspension`. -5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory - `NOT_RUN`, no `harness_errors`. -6. The armed run reports `PASS` for all four `ephemeral.*` checks, including - `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are - confirmed gone. -7. The JSON report is archived with the exact `origin/main` commit and the - recorded pull-request heads from `baseline.pending-pull-requests-recorded`. - -Only then is the handoff releasable. - -## Known limits - -Stated so nobody mistakes a proxy for the thing itself. - -* **Merge, approve, and close authority is asserted, not exercised.** The - harness never attempts a merge. It proves the forge identity holds - `permissions.admin: false` and is live-refused on an administrative route; - the route-level allowlist is covered by PR #14's own regression suite. -* **Attach and port-forward have no side-effect-free live attempt.** Their - authorization reviews ride alongside a real, refused control-plane write in the - same check, and pod exec — the one subresource with a harmless real attempt — - has its own check. -* **TokenRequest is asserted by review.** A successful live attempt would mint a - real credential, so the check pairs the `serviceaccounts/token` review with a - refused `serviceaccount` create in the same RBAC family. -* **Branch protection is not read.** The harness identity is not a repository - administrator, so `GET /branch_protections` is refused — which is itself - recorded as evidence. Protected-push refusal is guaranteed by the harness's own - preflight, exercised on every armed run, not by reading the forge's config. -* **Stale/duplicate assignment safety is asserted as deployed configuration.** - The behaviour itself is covered by the pool's regression suite; the harness - checks that the workers carry the configuration and that the coordinator keeps - sole ownership of its claims. -* **Telegram session continuity is per tenant** and depends on there having been - Telegram traffic, so it is off by default. The always-on check is that topic - and tenant state live on a persistent claim rather than in the pod. -* **The routing window is a tail**, 600 records by default. A property that did - not occur inside that window is reported as missing rather than absent; widen - the window or wait for traffic rather than removing the assertion. - -## Tests - -``` -python3 -m pytest testing/tests/ -k handoff -python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ - -m pytest testing/tests/ -k handoff && python3 -m coverage report +```sh +python3 -m testing.quality_gate --profile local --build-dir build +python3 -m testing.quality_handoff_mutation +python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m ruff format --check \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m compileall -q scripts/ops testing +kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - +git diff --check origin/main...HEAD ``` -The suite covers parsing, redaction, timeouts and deadlines, partial failure, -zero state, dual-vantage mismatch, mandatory skips, catalog structure, and -ephemeral arming and cleanup — including the cases where cleanup cannot be -verified. +The deterministic mutation gate must kill every policy, redaction, verdict, +numeric, truncation, and cleanup mutant. The adversarial suite includes large +concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, +absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, +and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, +and bounded diff/full-tree secret scans must also pass. + +The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their +result must be identical whether synthetic runtime files exist or are absent; +tests must never read the mounted production token. + +## Release decision + +The handoff remains `NO_GO` until all dependency heads are merged, the exact +lineage check passes from an external operator, both default and armed reports +are complete, the branch and PR cleanup is verified, the canonical and +adversarial gates are green, and a human has reviewed PR #19 plus the artifacts. +The harness never merges, deploys, reconciles, publishes, or approves anything. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py index 7be6de29..6b35d49f 100755 --- a/scripts/ops/hermes_handoff_acceptance.py +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -11,8 +11,9 @@ unreachable vantage, a catalog whose evidence no longer exists, an expired deadline, or two vantages that turn out to be the same principal. Nothing here rounds an absence of evidence up to a pass. - scripts/ops/hermes_handoff_acceptance.py --context atlas-operator - scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + scripts/ops/hermes_handoff_acceptance.py --help + +The release runbook lists every required freshness and lineage input. Mutation lives behind a separate arming flag, needs an exact confirmation phrase and a unique ref, refuses protected push targets before any network @@ -41,17 +42,24 @@ from hermes_handoff_ephemeral import ( # noqa: E402 from hermes_handoff_exec import ( # noqa: E402 DEFAULT_COMMAND_TIMEOUT, DEFAULT_RUN_DEADLINE, - SELF, Runner, ) from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 -from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_policy import ( # noqa: E402 + ARMED, + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + READ_ONLY, +) from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 from hermes_handoff_run import ( # noqa: E402 build_catalog, build_report, resolve_vantages, unavailable_report, + validate_catalog, + validate_targets, ) EXIT_GO = 0 @@ -59,6 +67,26 @@ EXIT_NO_GO = 1 EXIT_USAGE = 2 +def dependency_head(value: str) -> tuple[int, str]: + """Parse one `PR=SHA` dependency-head binding.""" + try: + number, sha = value.split("=", 1) + parsed = int(number) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "dependency head must be PR=40-character-SHA" + ) from exc + if ( + parsed <= 0 + or len(sha) != 40 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise argparse.ArgumentTypeError( + "dependency head must be PR=lowercase-40-character-SHA" + ) + return (parsed, sha) + + def build_parser() -> argparse.ArgumentParser: """Return the command-line interface for the harness.""" parser = argparse.ArgumentParser( @@ -67,21 +95,41 @@ def build_parser() -> argparse.ArgumentParser: formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") - parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument( + "--context", help="Kubernetes context for the operator vantage." + ) parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") - parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") - parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") parser.add_argument( - "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + "--output", type=Path, help="Write the JSON report here instead of stdout." ) parser.add_argument( - "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + "--json-only", action="store_true", help="Suppress the human summary." ) parser.add_argument( - "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + "--command-timeout", + type=float, + default=DEFAULT_COMMAND_TIMEOUT, + help="Per-command seconds.", ) parser.add_argument( - "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + "--deadline", + type=float, + default=DEFAULT_RUN_DEADLINE, + help="Whole-run seconds.", + ) + parser.add_argument( + "--max-output-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help="Per-capture byte budget.", + ) + parser.add_argument( + "--concurrency", type=int, default=1, help="Validated scheduling bound." + ) + parser.add_argument( + "--baseline-commit", + default=Targets.baseline_commit, + help="Commit main must descend from.", ) parser.add_argument( "--dependency-pr", @@ -90,6 +138,31 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Pull request that must be merged before this release is certifiable (repeatable).", ) + parser.add_argument( + "--dependency-head", + type=dependency_head, + action="append", + default=None, + metavar="PR=SHA", + help="Exact current dependency PR head (repeat for every dependency).", + ) + parser.add_argument( + "--remote-main-sha", default="", help="Exact observed origin/main SHA." + ) + parser.add_argument( + "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA." + ) + parser.add_argument( + "--agent-image", + default="", + help="Exact agent :git--build-@sha256 identity.", + ) + parser.add_argument( + "--build-sha", default="", help="Exact source SHA used for the image build." + ) + parser.add_argument( + "--deployment-revision", default="", help="Exact running Deployment revision." + ) parser.add_argument( "--expected-suspension", action="append", @@ -97,10 +170,19 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAMESPACE/NAME", help="Flux object allowed to be suspended (repeatable).", ) - parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") - parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") parser.add_argument( - "--chat-config-revision", default="", help="Expected chat pod-template config revision." + "--node-count", + type=int, + default=3, + help="Nodes the fleet must cover (fixed at 3).", + ) + parser.add_argument( + "--pool-replicas", type=int, default=3, help="Expected pool worker count." + ) + parser.add_argument( + "--chat-config-revision", + default="", + help="Expected chat pod-template config revision.", ) parser.add_argument( "--chat-ordinal", @@ -111,6 +193,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--expect-telegram-sessions", action="store_true", + default=True, help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", ) parser.add_argument( @@ -131,11 +214,12 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Permit one ephemeral branch push and draft pull request, then remove both.", ) - armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") - armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") - armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") - armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") - armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + armed.add_argument( + "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}." + ) + armed.add_argument( + "--ephemeral-token", default="", help="Unique 8-64 character ref suffix." + ) return parser @@ -145,17 +229,23 @@ def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Target return Targets( namespace=arguments.namespace, baseline_commit=arguments.baseline_commit, - dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + dependency_pull_requests=tuple( + arguments.dependency_pr or defaults.dependency_pull_requests + ), + dependency_heads=tuple(arguments.dependency_head or ()), + remote_main_sha=arguments.remote_main_sha, + reviewed_head_sha=arguments.reviewed_head_sha, + agent_image=arguments.agent_image, + build_sha=arguments.build_sha, + deployment_revision=arguments.deployment_revision, expected_suspensions=tuple(arguments.expected_suspension or ()), node_count=arguments.node_count, pool_replicas=arguments.pool_replicas, chat_config_revision=arguments.chat_config_revision, chat_ordinal=arguments.chat_ordinal, expect_telegram_sessions=arguments.expect_telegram_sessions, - pool_worker_env=tuple(arguments.pool_worker_env or ()), + pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), max_evidence_age_seconds=arguments.max_evidence_age, - repo=arguments.repo, - remote=arguments.remote, now=now, ) @@ -186,14 +276,16 @@ def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: request = ArmRequest( - repo=arguments.repo, - remote=arguments.remote, + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, token=arguments.ephemeral_token, confirmation=arguments.confirm, - base=arguments.ephemeral_base, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, ) - preflight(request, arguments.repo) - vantage = vantages.get(SELF) or vantages["operator"] + preflight(request, EXPECTED_REPO, Path.cwd()) + vantage = vantages["operator"] return run_armed(runner, vantage, request) @@ -205,27 +297,64 @@ def main(argv: Sequence[str] | None = None) -> int: targets = targets_from_args(arguments, now) mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY - runner = Runner( - mode=mode, - command_timeout=arguments.command_timeout, - deadline_seconds=arguments.deadline, - max_bytes=arguments.max_output_bytes, - ) - vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) - - armed_results: list = [] + specs = build_catalog(targets) + startup_problems = [*validate_targets(targets), *validate_catalog(specs)] if arguments.arm_ephemeral_push: try: - armed_results = _armed_results(arguments, runner, vantages) + preflight( + ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ), + EXPECTED_REPO, + Path.cwd(), + ) except ArmingError as exc: - report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) - emit(report, arguments) - return EXIT_NO_GO + startup_problems.append(f"arming refused: {exc}") + if startup_problems: + report = unavailable_report( + mode, started_at, "; ".join(startup_problems), targets + ) + emit(report, arguments) + return EXIT_NO_GO - specs = build_catalog(targets) - report = build_report( - runner, targets, specs, vantages, records, mode, started_at, armed_results + try: + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + concurrency=arguments.concurrency, + ) + except ValueError as exc: + report = unavailable_report( + mode, started_at, f"invalid execution bound: {exc}", targets + ) + emit(report, arguments) + return EXIT_USAGE + vantages, records = resolve_vantages( + runner, targets, arguments.kubeconfig, arguments.context ) + report = build_report(runner, targets, specs, vantages, records, mode, started_at) + if arguments.arm_ephemeral_push: + if report.decision != GO: + report.harness_errors.append( + "armed mutation was not started because read-only acceptance did not GO" + ) + else: + armed_results = _armed_results(arguments, runner, vantages) + armed_ids = {result.spec.id for result in armed_results} + report.results = [ + result for result in report.results if result.spec.id not in armed_ids + ] + report.results.extend(armed_results) + report.results.sort(key=lambda result: result.spec.id) + report.finished_at = utc_now() offenders = emit(report, arguments) if offenders: return EXIT_NO_GO diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py index b59ff2b9..9dc5c98a 100644 --- a/scripts/ops/hermes_handoff_catalog.py +++ b/scripts/ops/hermes_handoff_catalog.py @@ -4,8 +4,8 @@ The catalog is data on purpose. A reviewer should be able to read what the release is asserted to satisfy without reading any plumbing, and the harness should be able to prove structural properties over every entry — both vantages -represented, no impersonation from outside a pod, no mutation outside a server -dry run — before it runs a single command. +represented, no impersonation from outside a pod, and no mutation command of +any kind — before it runs a single command. Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload is renamed, the affected checks report ``NOT_RUN`` because the evidence they @@ -19,7 +19,8 @@ from collections.abc import Sequence from dataclasses import dataclass, field from typing import Any -from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step +from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step +from hermes_handoff_policy import projection OPERATOR = "operator" SELF = "self" @@ -28,21 +29,38 @@ NODE = "node" CHAT = "chat" VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) -# `namespace/namesuspendType=Status,...` — one compact row per object. -FLUX_PROJECTION = ( - "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" - "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" -).replace("'", '"') +# `namegenerationobservedGenerationsuspendconditions`. +FLUX_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" + ).replace("'", '"') +) -ENV_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" -).replace("'", '"') +ENV_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" + ).replace("'", '"') +) -INIT_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" -).replace("'", '"') +INIT_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" + ).replace("'", '"') +) -NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +NODE_NAME_PROJECTION = projection( + "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +) + +POOL_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}" + "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}" + "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}" + ).replace("'", '"') +) PROVIDER_API_KEY_NAMES = ( "ANTHROPIC_API_KEY", @@ -56,7 +74,12 @@ PROVIDER_API_KEY_NAMES = ( "OPENAI_API_KEY", "OPENAI_BASE_URL", ) -FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") +FORGE_CREDENTIAL_NAMES = ( + "GITEA_PASSWORD", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_PASSWORD", +) CODEX_HEALTH = "/opt/data/provider-health/codex.json" CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" @@ -92,31 +115,44 @@ class Targets: pool_statefulset: str = "hermes-execution-worker" pool_replicas: int = 3 pool_selector: str = "app=hermes-execution-worker" - coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + coordinator_claims: tuple[str, ...] = ( + "hermes-agent-home", + "hermes-agent-workspace", + ) node_account: str = "hermes-agent" node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" - node_count: int = 1 + node_count: int = 3 repo: str = "atlas/titan-iac" remote: str = "origin" baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" - dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20) + dependency_heads: tuple[tuple[int, str], ...] = () + remote_main_sha: str = "" + reviewed_pr_number: int = 19 + reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance" + reviewed_head_sha: str = "" + agent_image: str = "" + build_sha: str = "" + deployment_revision: str = "" expected_suspensions: tuple[str, ...] = () harbor_namespace: str = "harbor" harbor_immutability_job: str = "hermes-agent-immutability" builder_namespace: str = "jenkins" builder_serviceaccount: str = "hermes-image-builder" builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" - builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + builder_capabilities: str = ( + 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + ) agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) chat_config_revision: str = "" - pool_worker_env: tuple[str, ...] = () + pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL") max_evidence_age_seconds: int = 86_400 routing_tail_lines: int = 600 routing_tail_bytes: int = 512 * 1024 listing_bytes: int = 512 * 1024 git_askpass: str = "/opt/coordinator/gitea_askpass.sh" - expect_telegram_sessions: bool = False + expect_telegram_sessions: bool = True now: Any = None extra: dict[str, Any] = field(default_factory=dict) @@ -165,25 +201,3 @@ def check( scope=scope, rationale=rationale, ) - - -def deny( - identifier: str, - title: str, - review: Sequence[str], - attempt: Sequence[str], - rationale: str, - group: str = "access-denied", -) -> CheckSpec: - """Return a deny check that pairs a self review with a real refused request.""" - return check( - identifier, - title, - group, - "denied", - [ - step("review", SELF, *review, kind=REVIEW), - step("attempt", SELF, *attempt, kind=ATTEMPT), - ], - rationale=rationale, - ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py index a863e0f3..df5c8117 100644 --- a/scripts/ops/hermes_handoff_checks_access.py +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 """Acceptance checks for what Hermes may and may not do to the cluster. -Every deny check pairs an in-pod authorization review with a real request that -must be refused, because a review reports what RBAC says and only an attempt -proves the API server enforces it. Mutating attempts go through a server dry -run: the API server authorises the request and discards it, so the refusal is -genuine and nothing is written either way. +Denials use Kubernetes authorization reviews only. Live denied attempts were +removed because a policy regression could turn a supposed probe into a real +credential read or mutation; the default harness never constructs such argv. """ from __future__ import annotations @@ -16,11 +14,9 @@ from hermes_handoff_catalog import ( SELF, Targets, check, - deny, step, ) -from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec -from hermes_handoff_policy import shell +from hermes_handoff_model import REVIEW, CheckSpec def _access(targets: Targets) -> list[CheckSpec]: @@ -31,22 +27,45 @@ def _access(targets: Targets) -> list[CheckSpec]: "No ClusterRoleBinding grants the agent service account cluster-admin", "access-denied", "names_absent", - [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + [ + step( + "bindings", + OPERATOR, + "kubectl", + "get", + "clusterrolebindings", + "-o", + "name", + ) + ], {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, - rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.", ), - deny( + check( "access.secrets-are-denied", "Hermes cannot read Secrets cluster-wide", - ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "get", + "secrets", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.", ), check( "access.service-account-tokens-are-denied", "Hermes cannot mint a service-account token", "access-denied", - "denied", + "review_denied", [ step( "review-token", @@ -72,119 +91,155 @@ def _access(targets: Targets) -> list[CheckSpec]: namespace, kind=REVIEW, ), - step( - "attempt", - SELF, - "kubectl", - "--namespace", - namespace, - "create", - "serviceaccount", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, - ), ], - rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.", ), - deny( + check( "access.impersonation-is-denied", "Hermes cannot impersonate another identity", - ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), - ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), - "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "impersonate", + "users", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="The default harness never constructs an impersonated request.", ), - deny( + check( "access.workload-mutation-is-denied", "Hermes cannot mutate its own workload", - ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), - ( - "kubectl", - "--namespace", - namespace, - "patch", - f"deploy/{targets.agent_deployment}", - "--type=merge", - "--patch", - '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', - "--dry-run=server", - ), - "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "patch", + "deployments", + "--namespace", + namespace, + kind=REVIEW, + ) + ], + rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.", ), - deny( + check( "access.pod-exec-is-denied", "Hermes cannot exec into an arbitrary pod", - ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), - ( - "kubectl", - "--namespace", - namespace, - "exec", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--", - *shell("path_readable", path="/tmp"), - ), - "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/exec", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="No cross-pod exec request is created by a read-only run.", ), check( "access.attach-portforward-and-kube-system-writes-are-denied", "Hermes holds no attach, port-forward, or control-plane write authority", "access-denied", - "denied", + "review_denied", [ - step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), - step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), - step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), step( - "attempt", + "review-attach", SELF, "kubectl", + "auth", + "can-i", + "create", + "pods/attach", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-portforward", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/portforward", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-create", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "configmaps", "--namespace", "kube-system", - "create", - "configmap", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, + kind=REVIEW, ), ], - rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + rationale="Every authority is reviewed without issuing the request it would authorize.", ), check( "access.required-reads-are-allowed", "Hermes retains the reads its operator role depends on", "access-allowed", "allowed", - [ - step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), - step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), - step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), - step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), - step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), - ], - ), - check( - "access.pod-logs-are-allowed", - "Hermes can still read pod logs", - "access-allowed", - "allowed", [ step( - "logs", + "watch-review", + SELF, + "kubectl", + "auth", + "can-i", + "watch", + "pods", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "pods", SELF, "kubectl", "--namespace", - targets.namespace, - "logs", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--tail", - "1", - record=False, - ) + namespace, + "get", + "pods", + "-o", + "name", + ), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step( + "deployments", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "deployments", + "-o", + "name", + ), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), ], ), check( @@ -193,8 +248,28 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "allowed", [ - step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), - step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step( + "kustomizations", + SELF, + "kubectl", + "get", + "kustomizations.kustomize.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + step( + "helmreleases", + SELF, + "kubectl", + "get", + "helmreleases.helm.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), ], ), check( @@ -203,8 +278,26 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "vantages_agree", [ - step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), - step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + step( + "operator", + OPERATOR, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + step( + "self", + SELF, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), ], {"steps": ("operator", "self")}, rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", @@ -219,7 +312,19 @@ def _gitops(targets: Targets) -> list[CheckSpec]: f"No unexpected {label} suspension or unhealthy reconciliation", "gitops", "flux_health", - [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + [ + step( + "objects", + OPERATOR, + "kubectl", + "get", + kind, + "--all-namespaces", + "-o", + FLUX_PROJECTION, + record=False, + ) + ], {"step": "objects", "expected_suspensions": targets.expected_suspensions}, ) for kind, label in ( diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py index a3dbaad7..102f93e4 100644 --- a/scripts/ops/hermes_handoff_checks_delivery.py +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -17,7 +17,6 @@ from __future__ import annotations from hermes_handoff_catalog import ( FORGE_CREDENTIAL_NAMES, INIT_NAME_PROJECTION, - NODE, NODE_NAME_PROJECTION, OPERATOR, SELF, @@ -26,10 +25,7 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import ATTEMPT, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell - -HOST_ETC = "/host-etc" -HOST_HOME = "/host-home" +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _forge(targets: Targets) -> list[CheckSpec]: @@ -44,14 +40,6 @@ def _forge(targets: Targets) -> list[CheckSpec]: {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", ), - check( - "forge.worker-holds-no-credential-file", - "The worker home has no stored Git credential file", - "forge", - "stdout_matches", - [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], - {"step": "credentials", "equals": "unreadable"}, - ), check( "forge.identity-is-not-an-administrator", "The forge identity the worker uses holds no administrative rights", @@ -93,117 +81,11 @@ def _forge(targets: Targets) -> list[CheckSpec]: ], rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", ), - check( - "forge.clone-and-fetch-still-work", - "The worker can still read the repository it is expected to work in", - "forge", - "allowed", - [ - step( - "ls-remote", - SELF, - *shell( - "git_ls_remote", - askpass=targets.git_askpass, - url=f"https://scm.bstein.dev/{targets.repo}.git", - ), - ) - ], - ), ] def _nodes(targets: Targets) -> list[CheckSpec]: - account = targets.node_account return [ - check( - "nodes.account-password-is-locked", - "The dedicated node account has no usable password", - "nodes", - "stdout_matches", - [ - step( - "lock", - NODE, - *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), - ) - ], - {"step": "lock", "equals": "locked"}, - rationale="The probe reports the lock marker only; the password hash is never read.", - ), - check( - "nodes.account-identity-is-dedicated", - "The node account is the dedicated Hermes identity, not a shared one", - "nodes", - "stdout_matches", - [ - step( - "identity", - NODE, - *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), - ) - ], - {"step": "identity", "equals": targets.node_account_identity}, - ), - check( - "nodes.account-holds-no-supplementary-groups", - "The node account joins no supplementary or administrative group", - "nodes", - "stdout_matches", - [ - step( - "groups", - NODE, - *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), - ) - ], - {"step": "groups", "equals": ""}, - rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", - ), - check( - "nodes.account-has-no-sudo-policy", - "No sudoers fragment names the node account", - "nodes", - "stdout_matches", - [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], - {"step": "sudoers", "equals": "absent"}, - ), - check( - "nodes.account-holds-a-single-authorized-key", - "Exactly one authorized key is installed for the node account", - "nodes", - "stdout_matches", - [ - step( - "keys", - NODE, - *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), - ) - ], - {"step": "keys", "equals": "1"}, - rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", - ), - *[ - check( - f"nodes.{label}-is-root-only", - f"{title} remains unreadable to the node account", - "nodes", - "stdout_matches", - [step("mode", NODE, *shell("path_mode", path=path))], - {"step": "mode", "equals": expected}, - ) - for label, title, path, expected in ( - ( - "k3s-credentials", - "The K3s admin kubeconfig", - f"{HOST_ETC}/rancher/k3s/k3s.yaml", - ["600 root:root", "absent"], - ), - ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), - ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), - ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), - ) - ], check( "nodes.hardening-covers-every-node", "The node account audit runs on every node in the cluster", @@ -224,7 +106,11 @@ def _nodes(targets: Targets) -> list[CheckSpec]: NODE_NAME_PROJECTION, ) ], - {"step": "coverage", "minimum": targets.node_count}, + { + "step": "coverage", + "minimum": targets.node_count, + "total_equals": targets.node_count, + }, rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", ), ] @@ -237,7 +123,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.builder-service-account-is-tokenless", "The image builder service account mounts no API token", "build", - "json_field", + "stdout_matches", [ step( "serviceaccount", @@ -249,10 +135,10 @@ def _build(targets: Targets) -> list[CheckSpec]: "serviceaccount", targets.builder_serviceaccount, "-o", - "json", + projection("jsonpath={.automountServiceAccountToken}"), ) ], - {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + {"step": "serviceaccount", "equals": "false"}, ), check( "build.builder-runs-unprivileged-with-exact-capabilities", @@ -276,7 +162,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.harbor-immutability-rule-is-applied", "The Harbor immutable-tag rule for the agent image has been applied", "build", - "json_numeric", + "lines_match", [ step( "job", @@ -288,35 +174,145 @@ def _build(targets: Targets) -> list[CheckSpec]: "job", targets.harbor_immutability_job, "-o", - "json", + projection("jsonpath={.status.succeeded}"), ) ], - {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", ), - check( - "build.deployed-image-is-digest-pinned", - "The running agent image is pinned by digest, not by a mutable tag", - "build", - "names_present", - [ - step( - "image", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"deploy/{targets.agent_deployment}", - "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', - ) - ], - {"step": "image", "contains": ("@sha256:",)}, - ), + _release_lineage(targets), ] +def _release_lineage(targets: Targets) -> CheckSpec: + """Bind the exact reviewed release from Git through the Ready pods.""" + container = targets.agent_container + namespace = targets.namespace + deployment = f"deploy/{targets.agent_deployment}" + deployment_projection = projection( + ( + "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}" + "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}' + ).replace("'", '"') + ) + pod_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + replicaset_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + flux_projection = projection( + ( + "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}" + "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + "{.status.lastAppliedRevision}" + ).replace("'", '"') + ) + return check( + "release.exact-lineage-is-running", + "Remote main, reviewed PR, image, build, Flux, and running revision are identical", + "release", + "release_lineage", + [ + step( + "remote-main", + OPERATOR, + "git", + "ls-remote", + targets.remote, + "refs/heads/main", + ), + step( + "reviewed-pr", + OPERATOR, + GITEA_CLIENT, + "GET", + f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}", + ), + step( + "build-source", + OPERATOR, + "git", + "rev-parse", + f"{targets.remote}/main", + ), + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + deployment, + "-o", + deployment_projection, + ), + step( + "pods", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + f"app={targets.agent_deployment}", + "-o", + pod_projection, + ), + step( + "replicasets", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "replicasets", + "--selector", + f"app={targets.agent_deployment}", + "-o", + replicaset_projection, + ), + step( + "flux", + OPERATOR, + "kubectl", + "--namespace", + "flux-system", + "get", + "kustomization/hermes", + "-o", + flux_projection, + ), + ], + { + "main_sha": targets.remote_main_sha, + "head_sha": targets.reviewed_head_sha, + "head_ref": targets.reviewed_head_ref, + "pr_number": targets.reviewed_pr_number, + "image": targets.agent_image, + "build_sha": targets.build_sha, + "deployment_revision": targets.deployment_revision, + }, + rationale="A PASS requires one exact release identity across every source and running object.", + ) + + def _reliability(targets: Targets) -> list[CheckSpec]: return [ check( @@ -370,4 +366,9 @@ def _reliability(targets: Targets) -> list[CheckSpec]: def delivery_checks(targets: Targets) -> list[CheckSpec]: """Return the forge, node, build, and reliability checks.""" - return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] + return [ + *_forge(targets), + *_nodes(targets), + *_build(targets), + *_reliability(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py index d1986976..7126036b 100644 --- a/scripts/ops/hermes_handoff_checks_platform.py +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -26,12 +26,22 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _baseline(targets: Targets) -> list[CheckSpec]: api = f"/api/v1/repos/{targets.repo}" + dependency_heads = dict(targets.dependency_heads) checks = [ + check( + "baseline.local-origin-main-is-current", + "The local origin/main ref equals the fixed remote-main SHA", + "baseline", + "stdout_matches", + [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")], + {"step": "main", "equals": targets.remote_main_sha}, + rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.", + ), check( "baseline.origin-main-descends-merged-work", "origin/main descends from the merged Hermes isolation baseline", @@ -55,7 +65,15 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "Exact open pull-request heads are recorded, not assumed", "baseline", "json_record", - [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + [ + step( + "open", + OPERATOR, + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=open&limit=50", + ) + ], { "step": "open", "record": { @@ -87,7 +105,14 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "baseline", "json_field", [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], - {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + { + "step": "pr", + "fields": { + "merged": True, + "base.ref": "main", + "head.sha": dependency_heads.get(number, ""), + }, + }, rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", ) for number in targets.dependency_pull_requests @@ -206,7 +231,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: "routing.switchyard-is-available", "Switchyard has a ready replica serving every model-call boundary", "routing", - "json_numeric", + "lines_match", [ step( "deployment", @@ -217,10 +242,10 @@ def _routing(targets: Targets) -> list[CheckSpec]: "get", switchyard, "-o", - "json", + projection("jsonpath={.status.readyReplicas}"), ) ], - {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, ), check( "routing.provider-effort-and-fallback-evidence", @@ -244,7 +269,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Codex boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), check( @@ -252,7 +285,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Claude boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), ] @@ -261,7 +302,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: def _scope_profile( targets: Targets, name: str, workload: str, container: str, expected: str ) -> CheckSpec: - projection = ( + profile_projection = projection( f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' ) @@ -280,7 +321,7 @@ def _scope_profile( "get", workload, "-o", - projection, + profile_projection, ) ], {"step": "profile", "equals": expected}, @@ -291,7 +332,11 @@ def _scope_profile( def _scopes(targets: Targets) -> list[CheckSpec]: return [ _scope_profile( - targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + targets, + "agent", + f"deploy/{targets.agent_deployment}", + targets.agent_container, + "agent", ), _scope_profile( targets, @@ -322,7 +367,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: "get", "ingress", "-o", - 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}' + ), ) ], {"step": "hosts", "minimum": 3}, @@ -332,4 +379,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: def platform_checks(targets: Targets) -> list[CheckSpec]: """Return the baseline, identity, routing, and scope checks.""" - return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] + return [ + *_baseline(targets), + *_identity(targets), + *_routing(targets), + *_scopes(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py index a2e1abea..eba7df6d 100644 --- a/scripts/ops/hermes_handoff_checks_workers.py +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -17,14 +17,15 @@ from __future__ import annotations from hermes_handoff_catalog import ( CHAT, INIT_NAME_PROJECTION, - NODE_NAME_PROJECTION, OPERATOR, + POOL_PROJECTION, SELF, Targets, check, ) from hermes_handoff_catalog import step as step from hermes_handoff_model import EPHEMERAL, CheckSpec +from hermes_handoff_policy import projection KANBAN_STATUSES = [ "archived", @@ -43,10 +44,10 @@ def _pool(targets: Targets) -> list[CheckSpec]: selector = targets.pool_selector return [ check( - "pool.three-workers-are-ready", - "The distributed execution pool has its full complement of ready workers", + "pool.three-workers-have-distinct-nodes-and-volumes", + "Exactly three Ready ordinal workers occupy distinct nodes and claims", "pool", - "json_numeric", + "pool_topology", [ step( "statefulset", @@ -57,25 +58,14 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": { - "status.readyReplicas": {"min": targets.pool_replicas}, - "spec.replicas": {"min": targets.pool_replicas}, - }, - }, - ), - check( - "pool.workers-occupy-distinct-nodes", - "Each pool worker runs on its own node", - "pool", - "distinct_count", - [ + projection( + "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace( + "'", '"' + ) + ), + ), step( - "nodes", + "workers", OPERATOR, "kubectl", "--namespace", @@ -85,61 +75,16 @@ def _pool(targets: Targets) -> list[CheckSpec]: "--selector", selector, "-o", - NODE_NAME_PROJECTION, - ) + POOL_PROJECTION, + ), ], { - "step": "nodes", - "minimum": targets.pool_replicas, - "total_equals": targets.pool_replicas, + "state_step": "statefulset", + "worker_step": "workers", + "replicas": targets.pool_replicas, + "name": targets.pool_statefulset, }, - rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", - ), - check( - "pool.workers-own-distinct-volumes", - "Each pool worker has its own durable workspace", - "pool", - "distinct_count", - [ - step( - "claims", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - "persistentvolumeclaims", - "--selector", - selector, - "-o", - "name", - ) - ], - {"step": "claims", "minimum": targets.pool_replicas}, - ), - check( - "pool.workers-are-tokenless", - "Pool workers mount no Kubernetes API token", - "pool", - "json_field", - [ - step( - "statefulset", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.pool_statefulset}", - "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": {"spec.template.spec.automountServiceAccountToken": False}, - }, - rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.", ), check( "pool.coordinator-retains-sole-state-ownership", @@ -156,7 +101,9 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "names": targets.coordinator_claims}, @@ -167,13 +114,7 @@ def _pool(targets: Targets) -> list[CheckSpec]: def _pool_env(targets: Targets) -> list[CheckSpec]: - """Return the assignment-safety check only when a site declares what to expect. - - A ``names_present`` assertion over an empty expectation would pass on any - workload at all, which is worse than not asserting: it reads as coverage. - """ - if not targets.pool_worker_env: - return [] + """Return the mandatory assignment-safety configuration check.""" return [ check( "pool.assignment-safety-knobs-are-configured", @@ -190,12 +131,40 @@ def _pool_env(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}' + ), ) ], {"step": "env", "names": targets.pool_worker_env}, rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", - ) + ), + check( + "pool.assignment-fieldrefs-are-bound", + "Worker node and ordinal identities come from fixed downward-API fields", + "pool", + "names_present", + [ + step( + "source", + OPERATOR, + "git", + "show", + f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml", + record=False, + ) + ], + { + "step": "source", + "names": ( + "name: HERMES_WORKER_ORDINAL", + "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']", + "name: HERMES_WORKER_NODE", + "fieldPath: spec.nodeName", + ), + }, + rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.", + ), ] @@ -226,7 +195,18 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "Durable sessions are listed with a preview, a source, and an id", "surfaces", "lines_match", - [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + [ + step( + "sessions", + SELF, + "hermes", + "sessions", + "list", + "--limit", + "20", + record=False, + ) + ], {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, rationale="A row whose leading column is blank is the session-render regression seen from the data side.", ), @@ -248,7 +228,10 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: INIT_NAME_PROJECTION, ) ], - {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + { + "step": "init", + "names": ("patch-web-session-activity", "patch-api-server-sessions"), + }, rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", ), check( @@ -286,62 +269,64 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "get", f"deploy/{targets.chat_router_deployment}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "contains": ("state",)}, rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", ), ] - if targets.expect_telegram_sessions: - checks.append( - check( - "surfaces.chat-telegram-sessions-are-continuous", - "The selected chat tenant still holds durable Telegram-sourced sessions", - "surfaces", - "lines_match", - [ - step( - "telegram", - CHAT, - "hermes", - "sessions", - "list", - "--source", - "telegram", - "--limit", - "20", - record=False, - ) - ], - {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, - rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", - ) + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", ) - if targets.chat_config_revision: - checks.append( - check( - "surfaces.chat-runs-the-telegram-topic-revision", - "The chat surface runs the durable Telegram topic configuration", - "surfaces", - "stdout_matches", - [ - step( - "revision", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.chat_statefulset}", - "-o", - "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", - ) - ], - {"step": "revision", "equals": targets.chat_config_revision}, - rationale="Telegram topic continuity is carried by the config revision the pod template pins.", - ) + ) + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + projection( + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}" + ), + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", ) + ) return checks diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py index 5456482b..fdcfdbad 100644 --- a/scripts/ops/hermes_handoff_ephemeral.py +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""The separately armed, self-cleaning mutation mode of the acceptance harness. - -Proving that a worker can still open a draft pull request means actually opening -one, which is the only part of this harness that writes anything. It is -therefore a distinct mode with a distinct policy: an explicit confirmation -phrase, a caller-supplied unique ref, a preflight that refuses a protected -target before any network call, and a cleanup whose *verification* is itself a -mandatory check. - -The push target is guarded, not the pull-request base. Opening a pull request -against `main` writes nothing to `main`; pushing to it does. The refusal is -checked by name and by pattern, and the arming self-test exercises it on every -armed run rather than trusting that it was checked once. -""" +"""Strictly preflighted, discoverable, and self-cleaning armed mode.""" from __future__ import annotations +import configparser import re from collections.abc import Sequence from dataclasses import dataclass +from pathlib import Path +from typing import Any from hermes_handoff_exec import Outcome, Runner, Vantage from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + GITEA_CLIENT, + arm_ephemeral_policy, + register_ephemeral_pull, +) +from hermes_handoff_rules import strict_json CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" REF_PREFIX = "ephemeral/hermes-handoff-acceptance" REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") DRAFT_TITLE_PREFIX = "WIP: " +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" PROTECTED_NAMES = frozenset( { "default", @@ -51,48 +52,113 @@ class ArmingError(ValueError): @dataclass(frozen=True) class ArmRequest: - """Everything the armed mode needs, and nothing it can infer for itself.""" + """Every fixed and caller-confirmed input to one armed run.""" repo: str remote: str token: str confirmation: str - base: str = "main" + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" @property def ref(self) -> str: - """Return the exact, unique branch this run is allowed to create.""" return f"{REF_PREFIX}/{self.token}" def normalise_branch(name: str) -> str: """Return a branch name stripped of ref decoration, for comparison.""" - trimmed = name.strip().removeprefix("refs/heads/").strip("/") - return trimmed.lower() + return name.strip().removeprefix("refs/heads/").strip("/").lower() def assert_push_target_allowed(ref: str) -> None: - """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + """Raise unless ``ref`` is the one ephemeral branch grammar.""" if normalise_branch(ref) in PROTECTED_NAMES: raise ArmingError(f"refusing to push to protected branch {ref!r}") if not REF_RE.fullmatch(ref): raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") -def preflight(request: ArmRequest, expected_repo: str) -> None: - """Validate an arming request before any network call is made.""" +def _bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = _bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / _bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = _bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = _bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = ( + _bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + ) + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(_bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" if request.confirmation != CONFIRMATION: raise ArmingError("confirmation phrase does not match") - if request.repo != expected_repo: - raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") if not TOKEN_RE.fullmatch(request.token): - raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") assert_push_target_allowed(request.ref) - if not normalise_branch(request.base): - raise ArmingError("a pull-request base branch is required") + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) -def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: +def _spec(identifier: str, title: str) -> CheckSpec: return CheckSpec( id=f"ephemeral.{identifier}", title=title, @@ -101,7 +167,6 @@ def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: steps=(), mandatory=True, scope=EPHEMERAL, - rationale=rationale, ) @@ -111,11 +176,10 @@ def _result( status: str, reason: str, outcomes: Sequence[Outcome] = (), - evidence: dict | None = None, - rationale: str = "", + evidence: dict[str, Any] | None = None, ) -> CheckResult: return CheckResult( - spec=_spec(identifier, title, rationale), + spec=_spec(identifier, title), status=status, reason=reason, evidence=evidence or {}, @@ -124,97 +188,271 @@ def _result( def _protected_refusal_result() -> CheckResult: - """Exercise the protected-branch guard on this run, not on a past one.""" - refused: list[str] = [] accepted: list[str] = [] for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): try: assert_push_target_allowed(candidate) except ArmingError: - refused.append(candidate) - else: - accepted.append(candidate) - evidence = {"refused": refused, "accepted": accepted} - if accepted: - return _result( - "protected-branch-refusal", - "Arming against a protected branch is refused before any network call", - FAIL, - f"the guard accepted {accepted}", - evidence=evidence, - ) + continue + accepted.append(candidate) return _result( "protected-branch-refusal", "Arming against a protected branch is refused before any network call", - PASS, - "every protected and malformed push target was refused", - evidence=evidence, + FAIL if accepted else PASS, + f"guard accepted {accepted}" + if accepted + else "every protected and malformed target was refused", + evidence={"accepted": accepted}, ) def _detail(outcome: Outcome) -> str: - """Return the most informative bounded explanation an outcome carries.""" - return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + return ( + outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + ) -def _pull_request_number(outcome: Outcome) -> int | None: - match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) - return int(match.group(1)) if match else None +def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: + if not isinstance(payload, dict): + return ["pull request is not an object"] + wanted = { + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha}, + "head": {"ref": request.ref, "sha": request.expected_head}, + } + problems: list[str] = [] + if ( + not isinstance(payload.get("number"), int) + or isinstance(payload.get("number"), bool) + or payload["number"] <= 0 + ): + problems.append("number is malformed") + for field in ("state", "draft", "merged"): + if ( + type(payload.get(field)) is not type(wanted[field]) + or payload.get(field) != wanted[field] + ): + problems.append(f"{field} does not match") + for group, values in (("base", wanted["base"]), ("head", wanted["head"])): + observed = payload.get(group) + if not isinstance(observed, dict): + problems.append(f"{group} is malformed") + continue + for field, value in values.items(): + if observed.get(field) != value: + problems.append(f"{group}.{field} does not match") + return problems -def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: - """Push a unique ephemeral ref, open a draft pull request, and remove both. +def _discover(outcome: Outcome, request: ArmRequest) -> tuple[list[int], str]: + if not outcome.ok: + return ([], f"pull-request discovery failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return ([], f"pull-request discovery returned malformed JSON: {exc}") + if not isinstance(payload, list): + return ([], "pull-request discovery did not return an array") + matches: list[int] = [] + for index, item in enumerate(payload): + if ( + not isinstance(item, dict) + or not isinstance(item.get("head"), dict) + or not isinstance(item.get("base"), dict) + ): + return ([], f"pull-request discovery row {index + 1} is malformed") + if item["head"].get("ref") != request.ref: + continue + problems = _pr_problems(item, request) + if problems: + return ([], f"discovered pull request is invalid: {', '.join(problems)}") + matches.append(item["number"]) + if len(matches) != 1: + return ( + matches, + f"expected one exact draft pull request, discovered {len(matches)}", + ) + return (matches, "") - Every step is checked and every artifact is removed. Cleanup runs even when - an earlier step failed, and the cleanup verification is reported as its own - mandatory check so a half-removed branch cannot pass as success. - """ - api = f"/api/v1/repos/{request.repo}" - results = [_protected_refusal_result()] + +def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]: + """Validate the same exact PR identity with state changed to closed.""" + problems = _pr_problems(payload, request) + if isinstance(payload, dict) and payload.get("state") == "closed": + problems = [ + problem for problem in problems if problem != "state does not match" + ] + elif "state does not match" not in problems: + problems.append("state is not closed") + return problems + + +def _cleanup( + runner: Runner, + vantage: Vantage, + request: ArmRequest, + api: str, + numbers: Sequence[int], + discovery_error: str = "", +) -> CheckResult: outcomes: list[Outcome] = [] + problems = [discovery_error] if discovery_error else [] + for number in sorted(set(numbers)): + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), + vantage, + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} close failed") + continue + try: + payload = strict_json(closed.stdout) + except ValueError: + problems.append(f"pull request #{number} close response is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} close response does not match") + deleted = runner.run( + (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage + ) + outcomes.append(deleted) + if not deleted.ok: + problems.append("ephemeral branch deletion failed") + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ok or remaining.stdout.strip(): + problems.append("ephemeral branch absence could not be verified") + for number in sorted(set(numbers)): + observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(observed) + if not observed.ok: + problems.append(f"pull request #{number} closure could not be verified") + continue + try: + payload = strict_json(observed.stdout) + except ValueError: + problems.append(f"pull request #{number} verification is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} is not exactly closed") + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL if problems else PASS, + "; ".join(problems) + if problems + else "the exact branch and draft pull request are gone", + outcomes, + {"ref": request.ref, "pull_requests": sorted(set(numbers))}, + ) - existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) - outcomes.append(existing) - if not existing.ok: - results.append( + +def run_armed( + runner: Runner, vantage: Vantage, request: ArmRequest +) -> list[CheckResult]: + """Create, discover, validate, and clean one exact ephemeral branch/PR.""" + guard = _protected_refusal_result() + if guard.status != PASS: + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", NOT_RUN, - f"could not confirm the ref is unused: {_detail(existing)}", - outcomes, - ) + "protected-ref guard failed", + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "protected-ref guard failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + api = f"/api/v1/repos/{request.repo}" + reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) + existing = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if not existing.ok or existing.stdout.strip(): + reason = ( + _detail(existing) if not existing.ok else "the ephemeral ref already exists" ) - return results - if existing.stdout.strip(): - results.append( + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", FAIL, - f"{request.ref} already exists on {request.remote}; pick a fresh token", - outcomes, - ) - ) - return results - - pushed = runner.run( - ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + reason, + [existing], + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "branch preflight failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + local_head = runner.run( + ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve ) - outcomes.append(pushed) - results.append( - _result( - "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", - PASS if pushed.ok else FAIL, - f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", - [pushed], - {"ref": request.ref}, + pushed = ( + runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if local_head.ok and local_head.stdout.strip() == request.expected_head + else Outcome( + argv=("git", "push"), + vantage=vantage.name, + error="local HEAD changed after static preflight", ) ) - - number: int | None = None - if pushed.ok: + verified = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + branch_ok = verified.ok and verified.stdout.split() == [ + request.expected_head, + f"refs/heads/{request.ref}", + ] + branch_result = _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if branch_ok else FAIL, + "the exact reviewed HEAD was pushed" + if branch_ok + else "push could not be verified exactly", + [local_head, pushed, verified], + {"ref": request.ref, "head": request.expected_head}, + ) + numbers: list[int] = [] + discovery_error = "" + create_outcomes: list[Outcome] = [] + if branch_ok: created = runner.run( ( GITEA_CLIENT, @@ -225,89 +463,33 @@ def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[Che "--field", f"head={request.ref}", "--field", - f"base={normalise_branch(request.base)}", + f"base={EXPECTED_BASE}", "--field", "body=Ephemeral acceptance probe. Closed and deleted by the harness.", ), vantage, + reserve_seconds=reserve, ) - outcomes.append(created) - number = _pull_request_number(created) if created.ok else None - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - PASS if created.ok and number else FAIL, - f"opened draft pull request #{number}" - if created.ok and number - else f"draft creation failed: {_detail(created)}", - [created], - {"pull_request": number}, - ) + discovered = runner.run( + (GITEA_CLIENT, "GET", f"{api}/pulls?state=all&limit=50"), + vantage, + reserve_seconds=reserve, ) + create_outcomes = [created, discovered] + numbers, discovery_error = _discover(discovered, request) + for number in numbers: + register_ephemeral_pull(number) else: - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - NOT_RUN, - "the ephemeral branch was never pushed", - ) - ) - - results.append(_cleanup(runner, vantage, request, api, number)) - return results - - -def _cleanup( - runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None -) -> CheckResult: - """Close the pull request, delete the ref, and prove both are gone.""" - outcomes: list[Outcome] = [] - problems: list[str] = [] - - if number is not None: - closed = runner.run( - (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage - ) - outcomes.append(closed) - if not closed.ok: - problems.append(f"pull request #{number} could not be closed") - - deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) - outcomes.append(deleted) - - remaining = runner.run( - ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage - ) - outcomes.append(remaining) - if not remaining.ran: - problems.append(f"could not verify removal: {_detail(remaining)}") - elif remaining.stdout.strip(): - problems.append(f"{request.ref} still exists on {request.remote}") - - if number is not None: - state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) - outcomes.append(state) - if not state.ok: - problems.append(f"could not confirm pull request #{number} is closed") - elif '"state":"open"' in state.stdout.replace(" ", ""): - problems.append(f"pull request #{number} is still open") - - if problems: - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - FAIL, - "; ".join(problems), - outcomes, - {"ref": request.ref, "pull_request": number}, - ) - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - PASS, - "the ephemeral ref and pull request are gone and their absence was re-checked", - outcomes, - {"ref": request.ref, "pull_request": number}, + discovery_error = "branch creation was not verified" + draft_result = _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if len(numbers) == 1 and not discovery_error else FAIL, + f"discovered exact draft pull request #{numbers[0]}" + if len(numbers) == 1 and not discovery_error + else discovery_error, + create_outcomes, + {"pull_requests": numbers}, ) + cleanup = _cleanup(runner, vantage, request, api, numbers, discovery_error) + return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py index 72e3558c..61e9e06b 100644 --- a/scripts/ops/hermes_handoff_evaluators.py +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -22,12 +22,15 @@ from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec from hermes_handoff_rules import ( EVALUATORS, Evaluation, + dotted, evaluate, evaluator, + exact_equal, lines, looks_denied, missing_steps, step_keys, + strict_json, ) __all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] @@ -41,19 +44,25 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return blocked attempts = step_keys(spec, ATTEMPT) if not attempts: - return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + return Evaluation( + NOT_RUN, "deny checks require a real attempt, not only a review" + ) evidence: dict[str, Any] = {} for key in step_keys(spec, REVIEW): verdict = outcomes[key].stdout.strip().lower() evidence[key] = verdict if verdict != "no": return Evaluation( - FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, ) for key in attempts: outcome = outcomes[key] if outcome.ok: - return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + return Evaluation( + FAIL, f"attempt {key} succeeded but must be refused", evidence + ) if not looks_denied(outcome): return Evaluation( NOT_RUN, @@ -64,6 +73,28 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return Evaluation(PASS, "review and live attempt were both refused", evidence) +@evaluator("review_denied") +def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when every side-effect-free authorization review says no.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + reviews = step_keys(spec, REVIEW) + if not reviews: + return Evaluation(NOT_RUN, "a denial check requires an authorization review") + evidence: dict[str, str] = {} + for key in reviews: + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + return Evaluation(PASS, "every authorization review was denied", evidence) + + @evaluator("allowed") def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every required step succeeded and every review said yes.""" @@ -84,7 +115,9 @@ def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluatio ) continue if not outcome.ok: - return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + return Evaluation( + FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence + ) evidence[step.key] = "succeeded" return Evaluation(PASS, "required access is present", evidence) @@ -93,7 +126,9 @@ def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: return {name for pattern in patterns for name in observed if pattern in name} -def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: +def _line_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[set[str], str] | Evaluation: key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: @@ -117,7 +152,9 @@ def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval ) evidence = {"observed_count": len(observed), "offenders": offenders} if offenders: - return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation( + FAIL, f"forbidden names present: {', '.join(offenders)}", evidence + ) return Evaluation(PASS, "no forbidden names are present", evidence) @@ -139,14 +176,18 @@ def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eva ) evidence = {"observed_count": len(observed), "missing": missing} if missing: - return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"required names absent: {', '.join(missing)}", evidence + ) if not observed: return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) return Evaluation(PASS, "every required name is present", evidence) @evaluator("stdout_matches") -def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_stdout_matches( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Compare a step's trimmed stdout against an expected value or set.""" blocked = missing_steps(spec, outcomes) if blocked: @@ -160,12 +201,16 @@ def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev allowed = expected if isinstance(expected, list) else [expected] evidence = {"observed": actual} if actual not in allowed: - return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation( + FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence + ) return Evaluation(PASS, "output matches the expected value", evidence) @evaluator("distinct_count") -def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_distinct_count( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass when a step yields enough distinct rows — and, optionally, exactly N. Distinctness is the point for placement checks: three worker pods pinned to @@ -182,16 +227,24 @@ def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev values = lines(outcome) distinct = sorted(set(values)) minimum = spec.expect["minimum"] - evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + evidence = { + "total": len(values), + "distinct": len(distinct), + "values": distinct[:16], + } if not values: return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) if len(distinct) < minimum: return Evaluation( - FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + FAIL, + f"{len(distinct)} distinct values, expected at least {minimum}", + evidence, ) total_equals = spec.expect.get("total_equals") if total_equals is not None and len(values) != total_equals: - return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation( + FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence + ) return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) @@ -217,16 +270,22 @@ def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu evidence = {"rows": len(rows), "offenders": offenders[:8]} if len(rows) < spec.expect.get("minimum", 1): return Evaluation( - NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " - f"{spec.expect.get('minimum', 1)}", evidence + NOT_RUN, + f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", + evidence, ) if offenders: - return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation( + FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence + ) return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) @evaluator("vantages_agree") -def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_vantages_agree( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass only when two independent observations report the same fact. A disagreement is a finding in its own right: it usually means the manifest @@ -245,9 +304,13 @@ def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev observations[key] = sorted(lines(outcome)) evidence = {key: value[:16] for key, value in observations.items()} if observations[left_key] != observations[right_key]: - return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + return Evaluation( + FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence + ) if not observations[left_key]: - return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation( + NOT_RUN, "both vantages returned nothing to compare", evidence + ) return Evaluation(PASS, "both vantages report the same state", evidence) @@ -261,6 +324,172 @@ def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evalua ) +# This validator is one compact, linear state machine so every exit observes +# only fully validated state. +# fmt: off +@evaluator("pool_topology") +def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Require exactly three Ready ordinal workers on distinct nodes and claims.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") + if len(state) != 4: + return Evaluation(NOT_RUN, "StatefulSet projection is malformed") + expected = spec.expect.get("replicas", 3) + try: + replicas, ready, current = (int(value) for value in state[:3]) + except ValueError: + return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") + evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, + "automountServiceAccountToken": state[3]} + if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): + return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) + workers: dict[int, tuple[str, str]] = {} + names: list[str] = [] + for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): + columns = row.split("\t") + if len(columns) != 5: + return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) + name, node, phase, ready_state, claims = columns + match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) + claim_list = [claim for claim in claims.split(",") if claim] + if (not match or not node or phase != "Running" or ready_state != "True" + or len(claim_list) != 1): + return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) + ordinal = int(match.group(1)) + if ordinal in workers: + return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) + workers[ordinal] = (node, claim_list[0]) + names.append(name) + evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), + "claims": sorted({v[1] for v in workers.values()})}) + if set(workers) != set(range(expected)): + return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) + if (len({v[0] for v in workers.values()}) != expected + or len({v[1] for v in workers.values()}) != expected): + return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) + return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) +# fmt: on + + +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_IMAGE_RE = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" +) + + +def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") + if not isinstance(payload, dict): + return Evaluation(NOT_RUN, f"{label} JSON is not an object") + return payload + + +# fmt: off +@evaluator("release_lineage") +def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + expected = spec.expect + main_sha = expected["main_sha"] + head_sha = expected["head_sha"] + image = expected["image"] + build_sha = expected["build_sha"] + revision = str(expected["deployment_revision"]) + digest = image.rsplit("@", 1)[-1] + image_match = _IMAGE_RE.fullmatch(image) + if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) + and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) + and image_match and image_match.group(1) == build_sha and build_sha == main_sha): + return Evaluation(NOT_RUN, "release lineage expectations are malformed") + remote = outcomes["remote-main"].stdout.strip().split() + if remote != [main_sha, "refs/heads/main"]: + return Evaluation(FAIL, "remote main does not match the fixed expected SHA", + {"remote_main": remote[:2]}) + pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") + if isinstance(pr, Evaluation): + return pr + fields = { + "number": expected["pr_number"], + "state": "open", + "draft": True, + "merged": False, + "base.ref": "main", + "base.sha": main_sha, + "head.ref": expected["head_ref"], + "head.sha": head_sha, + } + for path, wanted in fields.items(): + try: + actual = dotted(pr, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") + if not exact_equal(actual, wanted): + return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) + if outcomes["build-source"].stdout.strip() != build_sha: + return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") + deployment = outcomes["deployment"].stdout.strip().split("\t") + if len(deployment) != 6: + return Evaluation(NOT_RUN, "Deployment projection is malformed") + generation, observed, replicas, ready, deployed_revision, deployed_image = deployment + try: + current = (int(generation) > 0 and generation == observed + and int(replicas) > 0 and replicas == ready) + except ValueError: + return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") + if not current or (deployed_revision, deployed_image) != (revision, image): + return Evaluation(FAIL, "Deployment lineage does not match the release contract") + replica_hash = "" + for number, row in enumerate(lines(outcomes["replicasets"]), 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") + _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns + if rs_revision != revision: + continue + if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): + return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") + if replica_hash: + return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") + replica_hash = rs_hash + if not replica_hash: + return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") + pod_rows = lines(outcomes["pods"]) + if len(pod_rows) != int(replicas): + return Evaluation(FAIL, "running pod count does not match the Deployment") + pods: list[str] = [] + for number, row in enumerate(pod_rows, 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"pod row {number} is malformed") + name, phase, ready_state, pod_image, image_id, pod_hash = columns + if phase != "Running" or ready_state != "True": + return Evaluation(FAIL, f"pod {name or number} is not Ready") + if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): + return Evaluation(FAIL, f"pod {name or number} lineage does not match") + pods.append(name) + flux = outcomes["flux"].stdout.strip().split("\t") + if len(flux) != 6: + return Evaluation(NOT_RUN, "Flux lineage projection is malformed") + name, generation, observed, suspended, ready_state, applied = flux + if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): + return Evaluation(FAIL, "Flux source is not current and Ready") + match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) + if not match or match.group(1) != main_sha: + return Evaluation(FAIL, "Flux applied revision is not the expected remote main") + return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", + {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, + "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) +# fmt: on + + # Two observations of the same fact agree whether or not they came from # different vantages; the dual-vantage case is just the important one. EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py index 200d82e0..08888b13 100644 --- a/scripts/ops/hermes_handoff_exec.py +++ b/scripts/ops/hermes_handoff_exec.py @@ -1,57 +1,100 @@ #!/usr/bin/env python3 -"""Bounded, screened command execution for the handoff acceptance harness. +"""Bounded, screened, provenance-attested command execution. -Every probe runs through :class:`Runner`, which owns the three budgets a -long-running acceptance sweep needs: a per-command timeout, a whole-run -deadline, and a per-capture byte budget. Exhausting the deadline does not -silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a -NO_GO, so a slow cluster can never be mistaken for a clean one. - -Subprocesses never inherit the caller's environment. A probe that echoed -`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child -environment is rebuilt from an allowlist of names whose values the harness -already needs to reach the cluster and the forge. +It bounds pipes, times out and reaps the process group, and discards truncated output. """ from __future__ import annotations +import hashlib +import math import os +import selectors import shlex +import shutil +import signal +import stat import subprocess import time from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress from dataclasses import dataclass, field +from pathlib import Path +from typing import Any -from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv -from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text +from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import ( + DEFAULT_MAX_BYTES, + TRUNCATION_NOTE, + redact, + safe_text, + scrub, +) DEFAULT_COMMAND_TIMEOUT = 25.0 DEFAULT_RUN_DEADLINE = 900.0 +DEFAULT_CLEANUP_RESERVE = 90.0 +MAX_COMMAND_TIMEOUT = 600.0 +MAX_RUN_DEADLINE = 7200.0 +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 +MAX_CONCURRENCY = 16 +READ_CHUNK = 64 * 1024 +PIPE_DRAIN_SECONDS = 0.25 -# Names whose values the harness needs to reach the cluster and the forge. The -# values are handed to the child process and are never recorded. -ENV_PASSTHROUGH = ( - "GIT_ASKPASS", - "GITEA_BASE_URL", - "HERMES_HOME", - "HOME", - "JENKINS_BASE_URL", - "KUBECONFIG", - "LANG", - "PATH", - "REQUESTS_CA_BUNDLE", - "SSL_CERT_FILE", - "VAULT_ADDR", -) +SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin" +# These exact allowlists are intentionally dense: reviewers compare them as a +# single provenance boundary rather than as extensible application config. +# fmt: off +ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE") +FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev", + "LC_ALL": "C", "PATH": SAFE_PATH} +EXPECTED_PATHS = { + "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, + "flux": {"/opt/data/tools/bin/flux", "/usr/local/bin/flux", "/usr/bin/flux"}, + "helm": {"/opt/data/tools/bin/helm", "/usr/local/bin/helm", "/usr/bin/helm"}, + "git": {"/usr/bin/git", "/bin/git"}, + "hermes": {"/opt/hermes/.venv/bin/hermes"}, + "sh": {"/usr/bin/sh", "/bin/sh"}, + GITEA_CLIENT: {GITEA_CLIENT}, +} +EXPECTED_SHA256 = { + "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"}, + "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"}, + "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"}, + "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"}, + GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"}, +} DEADLINE_ERROR = "deadline-exceeded" TIMEOUT_ERROR = "timeout" - OPERATOR = "operator" SELF = "self" - -# Tools that accept a `--context` flag directly after the binary name. CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") +POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", + "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} +# fmt: on + + +@dataclass(frozen=True) +class Attestation: + """The exact outer executable used for a probe.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class Capture: + """Raw bounded process result returned by an executor implementation.""" + + returncode: int + stdout: bytes = b"" + stderr: bytes = b"" + stdout_truncated: bool = False + stderr_truncated: bool = False + timed_out: bool = False @dataclass(frozen=True) @@ -66,116 +109,298 @@ class Outcome: truncated: bool = False duration_ms: int = 0 error: str | None = None + executable_path: str = "" + executable_sha256: str = "" @property def ok(self) -> bool: - """Report whether the command ran to completion and succeeded.""" - return self.error is None and self.returncode == 0 + return self.error is None and self.returncode == 0 and not self.truncated @property def ran(self) -> bool: - """Report whether the command reached the target at all.""" return self.error is None @property def combined(self) -> str: - """Return stdout and stderr joined, for message matching.""" return "\n".join(part for part in (self.stdout, self.stderr) if part) def as_dict(self) -> dict[str, object]: - """Return a JSON-serialisable view of the outcome.""" - return { - "command": redact(shlex.join(self.argv)), - "vantage": self.vantage, - "returncode": self.returncode, - "stdout": self.stdout, - "stderr": self.stderr, - "truncated": self.truncated, - "duration_ms": self.duration_ms, - "error": self.error, - } + return scrub( + { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + "executable_path": self.executable_path, + "executable_sha256": self.executable_sha256, + } + ) @dataclass(frozen=True) class Vantage: - """A named identity and transport a probe can be observed from. - - ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the - target container, so the inner command authenticates as the pod's own - service account instead of as the operator. ``inserts`` carries per-binary - flags — the operator's `--context`, which must not follow the command into - the pod. - """ + """A named identity and transport used to observe a probe.""" name: str prefix: tuple[str, ...] = () inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) env: Mapping[str, str] = field(default_factory=dict) + command_paths: Mapping[str, str] = field(default_factory=dict) description: str = "" def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: - """Return ``argv`` addressed to this vantage.""" command = tuple(argv) if command: + replacement = self.command_paths.get(command[0]) + if replacement: + command = (replacement, *command[1:]) extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) if extra: command = (command[0], *extra, *command[1:]) return (*self.prefix, *command) -def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: - """Return the child environment: allow-listed names plus vantage overrides.""" +def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + converted = float(value) + if not math.isfinite(converted) or not minimum <= converted <= maximum: + raise ValueError( + f"{name} must be finite and between {minimum:g} and {maximum:g}" + ) + return converted + + +def _byte_budget(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_OUTPUT_BYTES + ): + raise ValueError( + f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}" + ) + return value + + +def _concurrency(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_CONCURRENCY + ): + raise ValueError( + f"concurrency must be an integer between 1 and {MAX_CONCURRENCY}" + ) + return value + + +def build_environment( + vantage: Vantage, source: Mapping[str, str] | None = None +) -> dict[str, str]: + """Build a deterministic child environment without caller-controlled PATH.""" origin = os.environ if source is None else source environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} - environment.setdefault("LC_ALL", "C") + environment.update(FIXED_ENVIRONMENT) environment.update(vantage.env) + environment["PATH"] = SAFE_PATH + environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"] + environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"] return environment +# fmt: off def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: - """Return the external read-only operator vantage.""" - inserts = ( - {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} - ) - detail = " ".join( - part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part - ) - return Vantage( - name=OPERATOR, - inserts=inserts, - env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, - description=f"operator {detail}".strip(), - ) + inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {} + detail = " ".join(part for part in (f"context={context}" if context else "", + "kubeconfig-pinned" if kubeconfig else "") if part) + return Vantage(name=OPERATOR, inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip()) -def pod_vantage( - namespace: str, - pod: str, - container: str, - operator: Vantage | None = None, -) -> Vantage: - """Return the in-pod vantage: the inner command authenticates as the pod.""" +def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage: reach = operator.inserts.get("kubectl", ()) if operator else () - return Vantage( - name=SELF, - prefix=( - "kubectl", - *reach, - "exec", - "--namespace", - namespace, - pod, - "--container", - container, - "--", - ), - env=dict(operator.env) if operator else {}, - description=f"in-pod {namespace}/{pod}[{container}]", + return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace, + pod, "--container", container, "--"), + env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS, + description=f"in-pod {namespace}/{pod}[{container}]") +# fmt: on + + +def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation: + """Resolve and hash an executable from a fixed, trusted path set.""" + name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1] + candidate = ( + command if "/" in command else shutil.which(command, path=environment["PATH"]) ) + if not candidate: + raise OSError(f"trusted executable is unavailable: {name}") + resolved = str(Path(candidate).resolve(strict=True)) + allowed = { + str(Path(item).resolve(strict=False)) + for item in EXPECTED_PATHS.get(name, set()) + } + if resolved not in allowed: + raise OSError(f"executable resolved outside its trusted paths: {name}") + details = os.stat(resolved) + if ( + not stat.S_ISREG(details.st_mode) + or details.st_mode & 0o022 + or not details.st_mode & 0o111 + ): + raise OSError(f"trusted executable has unsafe ownership mode: {name}") + digest = hashlib.sha256() + with open(resolved, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + expected = EXPECTED_SHA256.get(name) + if not expected or observed not in expected: + raise OSError(f"executable digest is not release-attested: {name}") + return Attestation(resolved, observed) + + +def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool: + remaining = limit + 1 - len(buffer) + if remaining > 0: + buffer.extend(chunk[:remaining]) + return len(buffer) > limit or len(chunk) > remaining + + +def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signum) + + +def execute_bounded( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + clock: Callable[[], float] = time.monotonic, + popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen, +) -> Capture: + """Execute one process with bounded live pipe capture and group cleanup.""" + process = popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=dict(environment), + text=False, + close_fds=True, + start_new_session=True, + ) + streams = {"stdout": process.stdout, "stderr": process.stderr} + buffers = {"stdout": bytearray(), "stderr": bytearray()} + truncated = {"stdout": False, "stderr": False} + selector = selectors.DefaultSelector() + for name, stream in streams.items(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + expires = clock() + timeout + timed_out = False + try: + while process.poll() is None: + remaining = expires - clock() + if remaining <= 0: + timed_out = True + break + for key, _ in selector.select(min(remaining, 0.1)): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM) + drain_until = clock() + PIPE_DRAIN_SECONDS + while selector.get_map() and clock() < drain_until: + for key, _ in selector.select(max(0.0, drain_until - clock())): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL) + returncode = process.wait(timeout=PIPE_DRAIN_SECONDS) + finally: + selector.close() + for stream in streams.values(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + stream.close() + if ( + process.poll() is None + ): # pragma: no branch - fail-safe for wait/selector exceptions + _signal_group(process, signal.SIGKILL) + process.wait() + return Capture( + returncode=returncode, + stdout=bytes(buffers["stdout"][:max_bytes]), + stderr=bytes(buffers["stderr"][:max_bytes]), + stdout_truncated=truncated["stdout"], + stderr_truncated=truncated["stderr"], + timed_out=timed_out, + ) + + +def _injected_executor( + spawn: Callable[..., subprocess.CompletedProcess[Any]], +) -> Callable[..., Capture]: + """Adapt deterministic test doubles without weakening the production path.""" + + def execute( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> Capture: + completed = spawn( + list(argv), + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=dict(environment), + ) + stdout = str(completed.stdout or "").encode() + stderr = str(completed.stderr or "").encode() + return Capture( + completed.returncode, + stdout[:max_bytes], + stderr[:max_bytes], + len(stdout) > max_bytes, + len(stderr) > max_bytes, + ) + + return execute + + +def _screen_capture(data: bytes, truncated: bool, budget: int) -> str: + if truncated: + return TRUNCATION_NOTE.encode()[:budget].decode() + return safe_text(data.decode("utf-8", "replace"), budget)[0] class Runner: - """Run probe commands under policy, timeout, deadline, and byte budgets.""" + """Run probes under policy, provenance, timeout, deadline, and byte budgets.""" def __init__( self, @@ -183,70 +408,92 @@ class Runner: command_timeout: float = DEFAULT_COMMAND_TIMEOUT, deadline_seconds: float = DEFAULT_RUN_DEADLINE, max_bytes: int = DEFAULT_MAX_BYTES, + concurrency: int = 1, clock: Callable[[], float] = time.monotonic, - spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + executor: Callable[..., Capture] | None = None, + spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, environ: Mapping[str, str] | None = None, + attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable, ) -> None: self.mode = mode - self.command_timeout = command_timeout - self.max_bytes = max_bytes + self.command_timeout = _finite_number( + "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT + ) + deadline = _finite_number( + "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE + ) + self.max_bytes = _byte_budget(max_bytes) + self.concurrency = _concurrency(concurrency) self._clock = clock - self._spawn = spawn + self._executor = executor or ( + _injected_executor(spawn) if spawn else execute_bounded + ) self._environ = environ - self._deadline_at = clock() + deadline_seconds + self._attestor = attestor + self._deadline_at = clock() + deadline self.commands_run = 0 @property def remaining_seconds(self) -> float: - """Return the seconds left before the whole-run deadline.""" return self._deadline_at - self._clock() def run( - self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + self, + argv: Sequence[str], + vantage: Vantage, + max_bytes: int | None = None, + reserve_seconds: float = 0.0, ) -> Outcome: - """Run one command, returning a screened outcome and never raising.""" - budget_bytes = self.max_bytes if max_bytes is None else max_bytes + budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes) + reserve = _finite_number( + "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE + ) addressed = vantage.wrap(argv) try: check_argv(addressed, self.mode) except PolicyError as exc: return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") - - budget = min(self.command_timeout, self.remaining_seconds) + budget = min(self.command_timeout, self.remaining_seconds - reserve) if budget <= 0: return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) - + environment = build_environment(vantage, self._environ) + try: + attestation = self._attestor(addressed[0], environment) + except (OSError, ValueError) as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"attestation: {exc}" + ) + executed = (attestation.path, *addressed[1:]) + try: + check_argv(executed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") started = self._clock() self.commands_run += 1 try: - completed = self._spawn( - list(addressed), - capture_output=True, - text=True, - timeout=budget, - check=False, - env=build_environment(vantage, self._environ), - ) + capture = self._executor(executed, environment, budget, budget_bytes) except subprocess.TimeoutExpired: + capture = Capture(-signal.SIGKILL, timed_out=True) + except OSError as exc: return Outcome( argv=addressed, vantage=vantage.name, - error=TIMEOUT_ERROR, - duration_ms=int((self._clock() - started) * 1000), + error=f"spawn: {exc.strerror or type(exc).__name__}", + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) - except OSError as exc: - return Outcome( - argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" - ) - - stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) - stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + was_truncated = capture.stdout_truncated or capture.stderr_truncated + stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes) + stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes) return Outcome( argv=addressed, vantage=vantage.name, - returncode=completed.returncode, + returncode=capture.returncode, stdout=stdout, stderr=stderr, - truncated=cut_out or cut_err, + truncated=was_truncated, duration_ms=int((self._clock() - started) * 1000), + error=TIMEOUT_ERROR if capture.timed_out else None, + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py index 6c10e52d..0f44c6a2 100644 --- a/scripts/ops/hermes_handoff_json_rules.py +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -11,7 +11,8 @@ other rule: an absent path, an unparsable payload, or an empty result set is from __future__ import annotations import datetime as dt -import json +import math +import re from typing import Any from hermes_handoff_exec import Outcome @@ -20,11 +21,33 @@ from hermes_handoff_rules import ( Evaluation, dotted, evaluator, + exact_equal, is_list, missing_steps, parsed_step, + strict_json, ) +_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z") +_FLUX_NAME_RE = re.compile( + r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z" +) + + +def _timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str) or not value.strip(): + raise ValueError("timestamp is not a non-empty string") + observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if observed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return observed.astimezone(dt.timezone.utc) + + +def _now(value: Any) -> dt.datetime: + if not isinstance(value, dt.datetime) or value.tzinfo is None: + raise ValueError("run clock is not timezone-aware") + return value.astimezone(dt.timezone.utc) + @evaluator("json_field") def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: @@ -42,10 +65,12 @@ def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalua try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) evidence[path] = actual allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: + if not any(exact_equal(actual, item) for item in allowed): mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") if mismatches: return Evaluation(FAIL, "; ".join(mismatches), evidence) @@ -68,9 +93,17 @@ def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) - if not isinstance(actual, (int, float)) or isinstance(actual, bool): - return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + if ( + not isinstance(actual, (int, float)) + or isinstance(actual, bool) + or not math.isfinite(float(actual)) + ): + return Evaluation( + NOT_RUN, f"path {path} is not numeric in step {key}", evidence + ) evidence[path] = actual if "min" in bounds and actual < bounds["min"]: problems.append(f"{path}={actual} below minimum {bounds['min']}") @@ -96,23 +129,30 @@ def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu if isinstance(parsed, Evaluation): return parsed payload, key = parsed - now = spec.expect["now"] + try: + now = _now(spec.expect["now"]) + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid run clock: {exc}") evidence: dict[str, Any] = {} stale: list[str] = [] for path, max_age in spec.expect["fields"].items(): try: raw = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) try: - observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) - if observed.tzinfo is None: - observed = observed.replace(tzinfo=dt.timezone.utc) + observed = _timestamp(raw) + except (TypeError, ValueError): + return Evaluation( + NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence + ) age = (now - observed).total_seconds() evidence[path] = {"age_seconds": int(age)} - if age > max_age: + if age < 0: + stale.append(f"{path} is {abs(int(age))}s in the future") + elif age > max_age: stale.append(f"{path} is {int(age)}s old, limit {max_age}s") if stale: return Evaluation(FAIL, "; ".join(stale), evidence) @@ -161,18 +201,24 @@ def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval except (KeyError, ValueError, IndexError): actual = None allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: - problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + if not any(exact_equal(actual, item) for item in allowed): + problems.append( + f"{identity}: {path}={actual!r} expected one of {allowed!r}" + ) evidence["examined"] = seen[:32] if wanted is not None: absent = [item for item in wanted if item not in seen] if absent: - return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + return Evaluation( + NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence + ) elif not seen: return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) if problems: return Evaluation(FAIL, "; ".join(problems[:8]), evidence) - return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + return Evaluation( + PASS, f"{len(seen)} element(s) match the release contract", evidence + ) @evaluator("json_record") @@ -195,21 +241,23 @@ def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu try: evidence[label] = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) return Evaluation(PASS, "state recorded", evidence) - @evaluator("routing_evidence") -def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_routing_evidence( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Summarise a Switchyard routing log tail into provider/effort coverage. Route identifiers are ``///``, so provider, model family, effort tier, and lane all come from the same field. Fallback and failure evidence is a separate ``fallback_reason`` on the record that - was retried. The parser tolerates a clipped first line: a tail of a growing - file starts mid-record, and discarding one row is better than a NOT_RUN that - reads as an outage. + was retried. Every row must be complete strict JSON; a clipped or malformed + line makes the evidence unusable rather than being silently discarded. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -218,22 +266,64 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") - records = [] - for line in outcome.stdout.splitlines(): + records: list[dict[str, Any]] = [] + routes: list[list[str]] = [] + stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {} + now_value = spec.expect.get("now") + try: + now = _now(now_value) if now_value is not None else None + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}") + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): line = line.strip() if not line: continue try: - records.append(json.loads(line)) - except ValueError: - continue - routes = [ - str(record.get("model") or "").split("/") - for record in records - if str(record.get("model") or "").count("/") == 3 - ] + record = strict_json(line) + except ValueError as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}" + ) + if not isinstance(record, dict): + return Evaluation(NOT_RUN, f"routing row {row_number} is not an object") + try: + observed = _timestamp(record.get("ts")) + except (TypeError, ValueError) as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}" + ) + if now is not None and observed > now: + return Evaluation(FAIL, f"routing row {row_number} has a future timestamp") + model = record.get("model") + if not isinstance(model, str) or not model: + return Evaluation(NOT_RUN, f"routing row {row_number} has no model") + route = model.split("/") + if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route): + routes.append(route) + for label, value in ( + ("provider", route[1]), + ("effort", route[3]), + ("lane", route[0]), + ): + key_dimension = (label, value) + stamps_by_dimension[key_dimension] = max( + observed, stamps_by_dimension.get(key_dimension, observed) + ) + elif record.get("tier") != "classifier": + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed route {model!r}" + ) + fallback = record.get("fallback_reason") + if fallback is not None and ( + not isinstance(fallback, str) or not fallback.strip() + ): + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed fallback_reason" + ) + record["_observed"] = observed + records.append(record) fallbacks = [record for record in records if record.get("fallback_reason")] - stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + stamps = sorted(record["_observed"] for record in records) evidence: dict[str, Any] = { "records": len(records), "routed": len(routes), @@ -242,11 +332,15 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> "families": sorted({route[2] for route in routes}), "efforts": sorted({route[3] for route in routes}), "fallback_events": len(fallbacks), - "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), - "latest_ts": stamps[-1] if stamps else "", + "fallback_reasons": sorted( + {str(record["fallback_reason"]) for record in fallbacks} + ), + "latest_ts": stamps[-1].isoformat() if stamps else "", } if not records: - return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + return Evaluation( + NOT_RUN, "routing log tail contained no parsable records", evidence + ) missing = [ f"{label}:{item}" for label, wanted in ( @@ -260,18 +354,28 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> if spec.expect.get("require_fallback_evidence") and not fallbacks: missing.append("fallback:none-observed") max_age = spec.expect.get("max_age_seconds") - now = spec.expect.get("now") if max_age and now: - try: - newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) - age = (now - newest).total_seconds() - evidence["age_seconds"] = int(age) - if age > max_age: - missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + ages: dict[str, int] = {} + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ): + for item in wanted: + stamp = stamps_by_dimension.get((label, item)) + if stamp is None: + continue + age = int((now - stamp).total_seconds()) + ages[f"{label}:{item}"] = age + if age > max_age: + missing.append( + f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s" + ) + evidence["age_seconds"] = ages if missing: - return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"routing evidence missing {', '.join(missing)}", evidence + ) return Evaluation(PASS, "routing evidence covers the required lanes", evidence) @@ -279,14 +383,10 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Classify a compact Flux projection into suspensions and unhealthy objects. - Each row is ``namespace/namesuspendType=Status,...``. An object is - unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no - Healthy condition and no in-flight ``Reconciling=True``. That last clause - matters: Flux flips Ready to False for the duration of a reconcile, so - without it the verdict depends on when the snapshot was taken. Suspended - objects are excluded from the health tally and judged against the - expected-suspension list instead, because a deliberate park is a different - fact from a broken reconcile. + Each row carries name, generation, observed generation, suspension, and all + condition states. A non-suspended object must be current, explicitly + ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended + objects are judged against the exact expected-suspension list. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -299,23 +399,62 @@ def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu suspended: list[str] = [] unhealthy: list[str] = [] total = 0 - for line in outcome.stdout.splitlines(): + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): if not line.strip(): continue + columns = line.split("\t") + if len(columns) != 5: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has {len(columns)} columns" + ) + name, generation, observed_generation, suspend, conditions = columns + name = name.strip() + if not _FLUX_NAME_RE.fullmatch(name): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed object name" + ) + try: + current_generation = int(generation) + reconciled_generation = int(observed_generation) + except ValueError: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed generations" + ) + if current_generation <= 0 or current_generation != reconciled_generation: + return Evaluation( + FAIL, f"Flux row {row_number} has stale observedGeneration" + ) + if suspend.strip().lower() not in {"", "false", "true"}: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed suspension state" + ) total += 1 - name, _, rest = line.partition("\t") - suspend, _, conditions = rest.partition("\t") if suspend.strip().lower() == "true": - suspended.append(name.strip()) + suspended.append(name) continue - states = dict( - pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair - ) + states: dict[str, str] = {} + for pair in filter(None, conditions.strip().strip(",").split(",")): + if pair.count("=") != 1: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition" + ) + condition, state = pair.split("=", 1) + if ( + not condition + or state not in {"True", "False", "Unknown"} + or condition in states + ): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition state" + ) + states[condition] = state healthy = states.get("Healthy") - reconciling = states.get("Reconciling") == "True" - ready_failed = states.get("Ready") == "False" - if healthy == "False" or (healthy is None and ready_failed and not reconciling): - unhealthy.append(name.strip()) + if ( + states.get("Ready") != "True" + or healthy in {"False", "Unknown"} + or states.get("Reconciling") == "True" + ): + unhealthy.append(name) unexpected = sorted(set(suspended) - expected) evidence = { "total": total, diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py index d34b9ebc..d35d39f5 100644 --- a/scripts/ops/hermes_handoff_model.py +++ b/scripts/ops/hermes_handoff_model.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_redaction import contains_credential_shape, scrub +from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub PASS = "PASS" FAIL = "FAIL" @@ -108,6 +108,8 @@ class CheckResult: @property def blocking(self) -> bool: """Report whether this result alone forces NO_GO.""" + if self.status not in STATUSES: + return True return self.spec.mandatory and self.status in {FAIL, NOT_RUN} def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: @@ -172,9 +174,17 @@ class Report: """Return the number of results in each status.""" tally = dict.fromkeys(STATUSES, 0) for result in self.results: - tally[result.status] = tally.get(result.status, 0) + 1 + if result.status in tally: + tally[result.status] += 1 return tally + @property + def invalid_statuses(self) -> list[str]: + """Return check ids whose status is outside the report schema.""" + return [ + result.spec.id for result in self.results if result.status not in STATUSES + ] + @property def blocking(self) -> list[CheckResult]: """Return every result that forces NO_GO, most severe first.""" @@ -187,7 +197,12 @@ class Report: @property def decision(self) -> str: """Return GO only when nothing mandatory failed, skipped, or errored.""" - if self.harness_errors or self.blocking: + if ( + not self.results + or self.invalid_statuses + or self.harness_errors + or self.blocking + ): return NO_GO return GO @@ -201,20 +216,23 @@ class Report: def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: """Return the machine-readable report.""" - return { - "harness": HARNESS, - "schema_version": SCHEMA_VERSION, - "mode": self.mode, - "started_at": self.started_at, - "finished_at": self.finished_at, - "decision": self.decision, - "counts": self.counts, - "baseline": scrub(self.baseline), - "vantages": [vantage.as_dict() for vantage in self.vantages], - "harness_errors": [scrub(error) for error in self.harness_errors], - "blocking": [result.spec.id for result in self.blocking], - "checks": [result.as_dict(include_outcomes) for result in self.results], - } + return scrub( + { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "invalid_statuses": self.invalid_statuses, + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + ) def render_summary(self, width: int = 88) -> str: """Return the concise human summary printed alongside the JSON.""" @@ -239,9 +257,11 @@ class Report: lines.append(" groups:") for name in sorted(groups): bucket = groups[name] - detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + detail = " ".join( + f"{status}={bucket[status]}" for status in STATUSES if bucket[status] + ) lines.append(f" {name:<18}{detail}") - return "\n".join(lines) + return safe_text("\n".join(lines), 32 * 1024)[0] def utc_now(clock: Any = None) -> str: diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py index 473dd782..fc1a7097 100644 --- a/scripts/ops/hermes_handoff_policy.py +++ b/scripts/ops/hermes_handoff_policy.py @@ -1,24 +1,8 @@ #!/usr/bin/env python3 """Fail-closed command policy for the Hermes handoff acceptance harness. -Two properties have to hold structurally, not by convention: - -* the default run mutates nothing, and -* no probe can pull a credential value into the report. - -Both are enforced here, where an argv becomes a subprocess, so a mistake in the -check catalog surfaces as a policy error when the catalog is built rather than -as a mutation or a leak when it runs. `--as` is rejected on the operator side: -impersonation is not evidence about what Hermes can do, and a harness able to -impersonate would be tempted to substitute it for a real self-probe. It stays -available to the inner command of a `kubectl exec`, where an identity is -probing its own refusal to impersonate rather than borrowing authority. - -Shell is reachable only through frozen templates. Free-form `sh -c` cannot be -screened reliably, but a few probes genuinely need field extraction (report that -a password field is locked without ever reading its hash) or an explicit helper -path, so those scripts are named, reviewed here, and parameter-validated instead -of assembled by callers. +Before spawn it admits only credential-safe, structurally read-only grammar; +shell is limited to frozen templates with absolute tool paths. """ from __future__ import annotations @@ -30,28 +14,42 @@ READ_ONLY = "read-only" ARMED = "ephemeral-armed" MODES = (READ_ONLY, ARMED) -# Rejected everywhere. Impersonation and inline credentials have no honest use -# in an acceptance probe. -FORBIDDEN_ARGS = ( - "--as", - "--as-group", - "--as-uid", - "--token", - "--password", - "--username", - "--client-key", - "--client-certificate", -) -# Reachable only from inside a pod, where an identity is probing its own -# refusal to impersonate rather than borrowing someone else's authority. -IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") -FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) GITEA_CLIENT = "/opt/coordinator/gitea_api.py" -ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) +EXPECTED_REPO = "atlas/titan-iac" +EXPECTED_REMOTE = "origin" +EXPECTED_BASE = "main" + +# fmt: off +FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate", + "--client-key", "--password", "--token", "--username") +# fmt: on +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset( + {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} +) +ALLOWED_BINARIES = frozenset( + {"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT} +) +TRUSTED_EXECUTABLE_ROOTS = ( + "/bin/", + "/usr/bin/", + "/usr/local/bin/", + "/opt/data/tools/bin/", + "/opt/hermes/.venv/bin/", + "/opt/coordinator/", +) +TRUSTED_INNER_PATHS = { + "/bin/sh": "sh", + "/usr/bin/git": "git", + "/usr/bin/sh": "sh", + "/usr/local/bin/kubectl": "kubectl", + "/opt/hermes/.venv/bin/hermes": "hermes", + GITEA_CLIENT: GITEA_CLIENT, +} -# Flags that consume the following token, so it is never a subcommand. VALUE_FLAGS = frozenset( { + "-C", "-c", "-l", "-n", @@ -76,50 +74,44 @@ KUBECTL_READ_SUBCOMMANDS = frozenset( { "api-resources", "api-versions", - "auth", - "cluster-info", - "config", - "describe", - "events", "explain", "get", - "logs", "top", "version", } ) -# Reachable only through a dry run: the API server authorises the request and -# discards it. A rejection is the evidence, and nothing is persisted either way. -KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( - {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} -) KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) -KUBECTL_SUBCOMMANDS = ( - KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"} +SECRET_RESOURCES = frozenset({"secret", "secrets"}) +UNSAFE_PROJECTION_PARTS = ( + ".args", + ".command", + ".data", + ".envfrom", + ".secretkeyref", + ".status.token", + ".stringdata", + ".valuefrom", + "clientconfig", + "password", + "privatekey", ) -DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") - -# Resources whose objects carry credential material. Reading them is permitted -# only in name-only output form, so a regression that grants access still cannot -# deliver a secret value into a report. -NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) +SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' +SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip FLUX_READ_SUBCOMMANDS = frozenset( - {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} + {"check", "events", "get", "stats", "trace", "tree", "version"} ) -HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +HELM_READ_SUBCOMMANDS = frozenset({"history", "list", "status", "version"}) GIT_READ_SUBCOMMANDS = frozenset( { "cat-file", - "config", "diff", - "fetch", "for-each-ref", "log", "ls-remote", "ls-tree", "merge-base", - "remote", "rev-list", "rev-parse", "show", @@ -127,60 +119,47 @@ GIT_READ_SUBCOMMANDS = frozenset( } ) GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) -GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GIT_FORCE_FLAGS = frozenset( + {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} +) +GIT_UNSAFE_FLAGS = ( + "-c", + "--config-env", + "--exec-path", + "--upload-pack", + "--receive-pack", +) GITEA_READ_METHODS = frozenset({"GET"}) -GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) - -# The Hermes CLI is the only honest source for durable session and Kanban -# activity, so a narrow read-only slice of it is on the allowlist. -HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) - -# Paths that hold credential material. A probe may prove one exists or is -# unreadable; it may never read it outside a frozen template. -SECRET_PATH_RE = re.compile( - r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" - r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +HERMES_READ_COMMANDS = frozenset( + {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)} ) +# These locations contain credential bytes even when a caller claims it will +# only inspect metadata. They are rejected for every argv, including a frozen +# shell rendering and the inner command of kubectl exec. +SECRET_PATH_RE = re.compile( + r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:" + r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|" + r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|" + r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml" + r")(?:$|[/\s'\"])", + re.IGNORECASE, +) _PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") _PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") +_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z") -# Frozen shell templates. Each exists because a plain argv cannot express the -# safety property its probe needs. SHELL_TEMPLATES: dict[str, str] = { - # Report an account's lock state without reading its password hash. - "account_lock_state": ( - 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + "env_names": ( + "/usr/bin/env | /usr/bin/sed -n " + "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort" ), - # Report the account record's non-secret identity fields. - "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', - # Report supplementary group names the account joins, never their members. - "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", - # Prove a path's presence and mode without reading its bytes. - "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', - # Prove whether a path is readable by this identity, without printing it. - "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', - # List environment variable names only. Values never leave the container. - "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", - # Report a JSON document's top-level key names without any of its values. - "json_keys": ( - 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' - ), - # Report allow-listed non-secret status fields from a provider health file. "json_fields": ( - 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' - 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" ' + "{path} {fields}" ), - # Count non-empty lines, for authorized-key inventories. - "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", - # Tail an append-only evidence log. The harness parses it; the container is - # not assumed to carry an interpreter. - "tail_lines": "tail -n {limit} {path}", - # Prove the worker can still read the repository. `kubectl exec` does not - # inherit the coordinator process environment, so the askpass helper is - # named explicitly rather than assumed; the helper reads runtime state and - # the credential never reaches this command line. - "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", + "tail_lines": "/usr/bin/tail -n {limit} {path}", } @@ -199,16 +178,42 @@ def template_parameters(template_name: str) -> set[str]: def render_shell(template_name: str, **parameters: str) -> str: """Return a frozen shell template with validated parameters substituted.""" required = template_parameters(template_name) - template = SHELL_TEMPLATES[template_name] if set(parameters) != required: - raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + raise PolicyError( + f"shell template {template_name} expects parameters {sorted(required)}" + ) for key, value in parameters.items(): if not _PARAMETER_RE.fullmatch(value): raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") - return template.format(**parameters) + if SECRET_PATH_RE.search(value): + raise PolicyError("shell templates may not name credential paths") + if ( + template_name == "json_fields" + and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS + ): + raise PolicyError("json_fields requested a credential-bearing or unknown field") + return SHELL_TEMPLATES[template_name].format(**parameters) _RENDERED: set[str] = set() +_APPROVED_PROJECTIONS: set[str] = set() +_ARMED_REF = "" +_ARMED_PULLS: set[int] = set() + + +# fmt: off +def arm_ephemeral_policy(ref: str) -> None: + global _ARMED_REF + if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref): + raise PolicyError("armed policy requires one exact ephemeral ref") + _ARMED_REF = ref + _ARMED_PULLS.clear() + + +def register_ephemeral_pull(number: int) -> None: + if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0: + raise PolicyError("ephemeral pull registration is malformed") + _ARMED_PULLS.add(number) def shell(template_name: str, **parameters: str) -> tuple[str, ...]: @@ -218,24 +223,56 @@ def shell(template_name: str, **parameters: str) -> tuple[str, ...]: return ("sh", "-c", script) +def projection(value: str) -> str: + normalized = value.lower().replace(" ", "") + if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection is not credential-safe") + if ".metadata.annotations" in normalized and not any(key in normalized for key in ( + "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")): + raise PolicyError("Kubernetes annotations are not on the reviewed allowlist") + if ".metadata.labels" in normalized and "pod-template-hash" not in normalized: + raise PolicyError("Kubernetes labels are not on the reviewed allowlist") + if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized: + raise PolicyError("Kubernetes projection may expose environment values") + _APPROVED_PROJECTIONS.add(normalized) + return value +# fmt: on + + def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: - """Yield positional arguments, skipping flags and the values they consume.""" + """Yield positional arguments, skipping flags and their values.""" index = start while index < len(argv): item = argv[index] if item == "--": return if item.startswith("-"): - if item in VALUE_FLAGS: - index += 2 - continue - index += 1 + index += 2 if item in VALUE_FLAGS else 1 continue yield item index += 1 -def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: +def _binary_name(value: str) -> str: + if value == GITEA_CLIENT or ( + value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py") + ): + return GITEA_CLIENT + if "/" not in value: + return value + mapped = TRUSTED_INNER_PATHS.get(value) + if mapped: + return mapped + if "/../" in value or value.endswith("/..") or "/./" in value: + raise PolicyError(f"executable path is not normalized: {value}") + if not value.startswith(TRUSTED_EXECUTABLE_ROOTS): + raise PolicyError(f"executable path is not trusted: {value}") + return value.rsplit("/", 1)[-1] + + +def _reject_forbidden_arguments( + argv: tuple[str, ...], allow_impersonation: bool +) -> None: permitted = IMPERSONATION_ARGS if allow_impersonation else () boundary = argv.index("--") if "--" in argv else len(argv) for argument in argv[:boundary]: @@ -244,47 +281,85 @@ def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool continue if argument == forbidden or argument.startswith(f"{forbidden}="): raise PolicyError(f"forbidden argument: {forbidden}") + if argument == "--raw" or argument.startswith("--raw="): + raise PolicyError("raw API reads are forbidden") + for argument in argv: + if argument == "--dry-run" or argument.startswith("--dry-run="): + raise PolicyError("dry-run mutation commands are forbidden") def _reject_secret_paths(argv: tuple[str, ...]) -> None: - boundary = argv.index("--") if "--" in argv else len(argv) - for argument in argv[:boundary]: - if SECRET_PATH_RE.search(argument): - raise PolicyError("probes may not name a credential path outside a frozen template") + if any(SECRET_PATH_RE.search(argument) for argument in argv): + raise PolicyError("probes may not name a credential path") -def _assert_name_only_output(argv: tuple[str, ...]) -> None: - if "--output=name" in argv or "-oname" in argv: - return - for index, item in enumerate(argv[:-1]): - if item in {"-o", "--output"} and argv[index + 1] == "name": - return - raise PolicyError("credential-bearing resources may only be read with -o name") +def _resource_name(value: str) -> str: + return value.lower().split("/", 1)[0].split(".", 1)[0] + + +def _output_value(argv: tuple[str, ...]) -> str: + """Return kubectl's output argument in a single canonical form.""" + for index, item in enumerate(argv[1:], 1): + if item in {"-o", "--output"} and index + 1 < len(argv): + return argv[index + 1] + if item.startswith(("-o=", "--output=")): + return item.split("=", 1)[1] + return "" + + +def _check_projection(output: str) -> None: + """Reject full or credential-bearing Kubernetes output projections.""" + lowered = output.lower().replace(" ", "") + if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"): + raise PolicyError( + "full Kubernetes object output is forbidden; use a safe projection" + ) + if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")): + raise PolicyError( + "Kubernetes output must be name or an approved JSONPath projection" + ) + if lowered.startswith("jsonpath="): + if any(part in lowered for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection may expose runtime credentials") + if ".env" in lowered and SAFE_ENV_VALUE not in lowered: + # Environment names are safe; arbitrary `.value` reads are not. + without_names = lowered.replace("{.name}", "") + if ".value" in without_names: + raise PolicyError("Kubernetes projection may expose environment values") + if lowered not in _APPROVED_PROJECTIONS: + raise PolicyError( + "Kubernetes JSONPath projection is not on the reviewed allowlist" + ) def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: words = list(positionals(argv)) - subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) - if subcommand is None: + if not words or words[0] not in KUBECTL_SUBCOMMANDS: raise PolicyError("kubectl subcommand is outside the read-only boundary") - if subcommand != words[0]: - raise PolicyError("kubectl subcommand must precede its resource arguments") - resource = words[1].split(".")[0] if len(words) > 1 else "" - if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: - _assert_name_only_output(argv) - if subcommand in KUBECTL_READ_SUBCOMMANDS: + subcommand = words[0] + if subcommand == "auth": + if len(words) < 2 or words[1] not in {"can-i", "whoami"}: + raise PolicyError("kubectl auth is limited to can-i and whoami") return - if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if subcommand == "exec": if "--" not in argv: raise PolicyError("kubectl exec requires an explicit -- command boundary") inner = argv[argv.index("--") + 1 :] if not inner: raise PolicyError("kubectl exec requires an inner command") + if inner[0] not in TRUSTED_INNER_PATHS: + raise PolicyError( + "kubectl exec inner command must use a fixed trusted path" + ) check_argv(inner, mode, allow_impersonation=True) return - if any(flag in argv for flag in DRY_RUN_FLAGS): - return - raise PolicyError(f"kubectl {subcommand} requires a dry run") + resources = words[1].split(",") if len(words) > 1 else [] + if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): + raise PolicyError("Secret objects may not be read by the harness") + output = _output_value(argv) + if subcommand == "get" and not output: + raise PolicyError("kubectl get requires a reviewed output projection") + _check_projection(output) def _first_positional(binary: str, argv: tuple[str, ...]) -> str: @@ -294,21 +369,39 @@ def _first_positional(binary: str, argv: tuple[str, ...]) -> str: return word -def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: +def _check_read_only_tool( + binary: str, argv: tuple[str, ...], allowed: frozenset[str] +) -> None: subcommand = _first_positional(binary, argv) if subcommand not in allowed: raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") def _check_git(argv: tuple[str, ...], mode: str) -> None: + unsafe = False + for item in argv[1:]: + if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")): + unsafe = True + if item == "-c" or (item.startswith("-c") and len(item) > 2): + unsafe = True + if any( + item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:] + ): + unsafe = True + if unsafe: + raise PolicyError("git configuration and helper overrides are forbidden") subcommand = _first_positional("git", argv) - if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): - raise PolicyError("git config may only read") if subcommand in GIT_READ_SUBCOMMANDS: return - if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if subcommand == "push" and mode == ARMED: if GIT_FORCE_FLAGS & set(argv): raise PolicyError("git push may not force or fan out across refs") + words = list(positionals(argv)) + if len(words) != 3 or words[1] != EXPECTED_REMOTE: + raise PolicyError("armed git push is pinned to origin and one refspec") + destination = words[2].partition(":")[2] + if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}": + raise PolicyError("armed git push is not the preflighted ephemeral ref") return raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") @@ -318,52 +411,76 @@ def _check_hermes(argv: tuple[str, ...]) -> None: if not words: raise PolicyError("hermes requires a subcommand") if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: - raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + raise PolicyError( + f"hermes {' '.join(words[:2])} is outside the read-only boundary" + ) def _check_gitea(argv: tuple[str, ...], mode: str) -> None: if len(argv) < 3: raise PolicyError("gitea_api.py requires a method and a path") - method = argv[1].upper() + method, path = argv[1].upper(), argv[2] + if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): + raise PolicyError("gitea_api.py requires a bounded API path") + if path != "/api/v1/user" and not path.startswith(f"/api/v1/repos/{EXPECTED_REPO}"): + raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") if method in GITEA_READ_METHODS: return - if method in GITEA_ARMED_METHODS and mode == ARMED: - return + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if mode == ARMED: + fields = [ + argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" + ] + if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF: + required = { + "title=WIP: Hermes handoff acceptance ephemeral probe", + f"head={_ARMED_REF}", + f"base={EXPECTED_BASE}", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + } + if len(fields) == len(required) and set(fields) == required: + return + match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path) + if ( + method == "PATCH" + and match + and int(match.group(1)) in _ARMED_PULLS + and fields == ["state=closed"] + ): + return + if ( + method == "DELETE" + and _ARMED_REF + and path == f"{repo_root}/branches/{_ARMED_REF}" + and len(argv) == 3 + ): + return raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") def _check_shell(argv: tuple[str, ...]) -> None: - if len(argv) != 3 or argv[1] != "-c": + if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED: raise PolicyError("sh is only reachable as sh -c ") - if argv[2] not in _RENDERED: - raise PolicyError("sh -c script is not a frozen template rendering") def check_argv( argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False ) -> None: - """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. - - ``allow_impersonation`` is set only for the inner command of a - ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; - the operator impersonating Hermes is not, so `--as` stays unreachable from - the outer, operator-side argv. - """ + """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" if mode not in MODES: raise PolicyError(f"unknown mode: {mode}") if not argv: raise PolicyError("empty command") - binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + binary = _binary_name(argv[0]) if binary in FORBIDDEN_BINARIES: raise PolicyError(f"forbidden binary: {binary}") if binary not in ALLOWED_BINARIES: raise PolicyError(f"binary is not on the harness allowlist: {binary}") _reject_forbidden_arguments(argv, allow_impersonation) + _reject_secret_paths(argv) if binary == "sh": _check_shell(argv) - return - _reject_secret_paths(argv) - if binary == "kubectl": + elif binary == "kubectl": _check_kubectl(argv, mode, allow_impersonation) elif binary == "flux": _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py index f48344a5..f7b97f4c 100644 --- a/scripts/ops/hermes_handoff_redaction.py +++ b/scripts/ops/hermes_handoff_redaction.py @@ -9,9 +9,10 @@ harness therefore also never asks for a secret in the first place. Probes read names, shapes, and status fields; they never read environment values, token files, private keys, cookies, or Vault responses. -Bounding is part of the same barrier. An unbounded capture is how a debug dump -or a stack trace carrying an authorization header ends up in an artifact, so -captures are truncated at a byte budget before they are screened. +Bounding is part of the same barrier. Screening always precedes truncation so a +credential that crosses the byte boundary cannot leave a useful prefix behind. +The process runner additionally discards all captured bytes when a live stream +exceeds its cap; the fixed truncation marker is the only recorded output. """ from __future__ import annotations @@ -62,12 +63,13 @@ SENSITIVE_KEY_EXACT = frozenset( # often than it is a credential, so it is screened by value shape and by the # token patterns rather than by name. The compound forms above still match. -# Values that cannot hold a credential: booleans, numbers, and anything too -# short to be one. Screening them corrupts structured evidence — a redacted -# `automountServiceAccountToken` is exactly the fact a reader needs — without -# protecting anything. +# Ordinary prose values shorter than this threshold are not independently +# treated as secret-shaped. Values under sensitive keys are screened at every +# length, including numeric values. MIN_SECRET_LENGTH = 8 -NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +NON_SECRET_LITERALS = frozenset( + {"true", "false", "null", "none", "yes", "no", "0", "1", ""} +) _KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") # Ordered: structural matches first so a nested token is not partially rewritten @@ -77,20 +79,23 @@ _PATTERNS: tuple[re.Pattern[str], ...] = ( r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", re.DOTALL, ), + re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL), re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), - re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile( + r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}" + ), re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), - re.compile(r"(? str: @@ -136,7 +143,7 @@ def _is_high_entropy(candidate: str) -> bool: any(character in "+/_=-" for character in candidate), ) ) - return classes >= 3 + return classes >= 4 def could_hold_secret(value: str) -> bool: @@ -153,13 +160,16 @@ def could_hold_secret(value: str) -> bool: def _redact_assignment(match: re.Match[str]) -> str: value = match.group("value") - if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + stripped = value.strip().strip("\"'").lower() + if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS: return match.group(0) # Preserve the original quoting so screened JSON stays parsable JSON. quoted = value.strip()[:1] in {'"', "'"} replacement = f'"{REDACTED}"' if quoted else REDACTED key_quote = match.group("quote") - return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + return ( + f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + ) def _redact_entropy(match: re.Match[str]) -> str: @@ -183,32 +193,42 @@ def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: encoded = text.encode("utf-8", "replace") if len(encoded) <= max_bytes: return (text, False) - kept = encoded[:max_bytes].decode("utf-8", "ignore") - return (f"{kept}\n{TRUNCATION_NOTE}", True) + marker = TRUNCATION_NOTE.encode()[:max_bytes] + content_budget = max( + 0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0) + ) + kept = encoded[:content_budget].decode("utf-8", "ignore") + separator = "\n" if kept and len(marker) < max_bytes else "" + return (f"{kept}{separator}{marker.decode()}", True) def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: - """Bound then redact, in that order, so a huge dump cannot outrun screening.""" - bounded, truncated = bound(text, max_bytes) - return (redact(bounded), truncated) + """Redact then bound, so truncation never exposes a credential prefix.""" + return bound(redact(text), max_bytes) def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: """Recursively bound and redact a JSON-shaped value. - Mapping keys are preserved (names are evidence); values under a sensitive - key are dropped outright rather than pattern-screened, because a short - secret would survive screening. + Mapping keys are themselves screened; values under a sensitive key are + dropped outright rather than pattern-screened, because a short secret + would survive screening. """ if isinstance(value, str): return safe_text(value, max_bytes)[0] if isinstance(value, dict): - return { - str(key): REDACTED - if is_sensitive_key(str(key)) and could_hold_secret(str(item)) - else scrub(item, max_bytes) - for key, item in value.items() - } + screened: dict[str, Any] = {} + for key, item in value.items(): + raw_key = str(key) + safe_key = safe_text(raw_key, max_bytes)[0] + sensitive_value = ( + is_sensitive_key(raw_key) + and item is not None + and not isinstance(item, bool) + and item != "" + ) + screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes) + return screened if isinstance(value, (list, tuple)): return [scrub(item, max_bytes) for item in value] return value diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py index cf9b5b97..5cca2c5a 100644 --- a/scripts/ops/hermes_handoff_rules.py +++ b/scripts/ops/hermes_handoff_rules.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_model import NOT_RUN, CheckSpec +from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec DENIAL_MARKERS = ( "forbidden", @@ -28,9 +28,7 @@ DENIAL_MARKERS = ( "cannot delete", "cannot patch", "not permitted", - "no such route", - "404 not found", - "403", + "gitea api returned http 403", ) @@ -57,7 +55,9 @@ def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: return register -def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: +def missing_steps( + spec: CheckSpec, outcomes: Mapping[str, Outcome] +) -> Evaluation | None: """Return NOT_RUN when a required step never produced usable evidence.""" for step in spec.steps: if step.optional: @@ -65,6 +65,8 @@ def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluatio outcome = outcomes.get(step.key) if outcome is None: return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if outcome.truncated: + return Evaluation(NOT_RUN, f"step {step.key} output was truncated") if not outcome.ran: return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") return None @@ -88,6 +90,11 @@ def is_list(value: Any) -> bool: return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) +def exact_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's bool/int type confusion.""" + return type(actual) is type(expected) and actual == expected + + def dotted(payload: Any, path: str) -> Any: """Return a value addressed by a dotted path. @@ -110,7 +117,9 @@ def dotted(payload: Any, path: str) -> Any: raise KeyError(path) if chunk == "": remainder = ".".join(segments[position + 1 :]) - return [dotted(item, remainder) if remainder else item for item in current] + return [ + dotted(item, remainder) if remainder else item for item in current + ] offset = int(chunk) if offset >= len(current): raise KeyError(path) @@ -118,17 +127,34 @@ def dotted(payload: Any, path: str) -> Any: return current -def parse_json(outcome: Outcome) -> Any: - """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. - - Control characters are tolerated. Task bodies and session titles routinely - carry raw newlines and tabs, and refusing to read a report because a human - pasted a tab into a Kanban card would be a false NO_GO. - """ - text = outcome.stdout.strip() +def strict_json(text: str) -> Any: + """Decode strict JSON, rejecting constants and duplicate object keys.""" + text = text.strip() if not text: raise ValueError("empty output") - return json.loads(text, strict=False) + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + return json.loads( + text, + strict=True, + parse_constant=reject_constant, + object_pairs_hook=unique_object, + ) + + +def parse_json(outcome: Outcome) -> Any: + """Parse one outcome as strict JSON.""" + return strict_json(outcome.stdout) def lines(outcome: Outcome) -> list[str]: @@ -136,7 +162,9 @@ def lines(outcome: Outcome) -> list[str]: return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] -def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: +def parsed_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[Any, str] | Evaluation: """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" key = spec.expect["step"] outcome = outcomes[key] @@ -153,7 +181,19 @@ def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: rule = EVALUATORS.get(spec.rule) if rule is None: return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated) + if truncated: + return Evaluation( + NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}" + ) try: - return rule(spec, outcomes) + result = rule(spec, outcomes) except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS - return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}" + ) + if result.status not in STATUSES: + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}" + ) + return result diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py index cdfaf3c2..a3b21cd7 100644 --- a/scripts/ops/hermes_handoff_run.py +++ b/scripts/ops/hermes_handoff_run.py @@ -4,8 +4,8 @@ Three things happen before any check runs, and each of them can stop the run on its own: -* the catalog is validated structurally — every rule exists, every deny check - carries a real attempt, no operator-side step can impersonate; +* the catalog is validated structurally — every rule exists, every live-denial + evaluator carries a read-only attempt, no operator-side step can impersonate; * each vantage is resolved and asked who it is; and * the operator and in-pod identities are compared. If they are the same principal there is only one vantage, the dual-vantage evidence would be a @@ -19,7 +19,7 @@ which is exactly the fail-closed answer. from __future__ import annotations -import json +import re from collections.abc import Iterable, Mapping, Sequence from hermes_handoff_catalog import ( @@ -48,7 +48,14 @@ from hermes_handoff_model import ( VantageRecord, utc_now, ) -from hermes_handoff_policy import IMPERSONATION_ARGS +from hermes_handoff_policy import ( + EXPECTED_REMOTE, + EXPECTED_REPO, + IMPERSONATION_ARGS, + PolicyError, + check_argv, + projection, +) UNRECORDED = "[output not recorded]" @@ -66,41 +73,129 @@ def build_catalog(targets: Targets) -> list[CheckSpec]: def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: """Return the structural problems that make a catalog unsafe to run.""" problems: list[str] = [] + if not specs: + return ["catalog is empty"] seen: set[str] = set() + fake_operator = operator_vantage() + fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) for spec in specs: if spec.id in seen: problems.append(f"{spec.id}: duplicate check id") seen.add(spec.id) if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + step_names: set[str] = set() for step in spec.steps: + if step.key in step_names: + problems.append(f"{spec.id}: duplicate step key {step.key!r}") + step_names.add(step.key) if step.vantage not in VANTAGE_NAMES: - problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + problems.append( + f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" + ) if step.kind not in STEP_KINDS: - problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + problems.append( + f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" + ) if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): - problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") - if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): - problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + problems.append( + f"{spec.id}: step {step.key} impersonates from the operator vantage" + ) + try: + addressed = ( + fake_operator if step.vantage == OPERATOR else fake_pod + ).wrap(step.argv) + check_argv(addressed) + except PolicyError as exc: + problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") + if spec.rule == "denied" and not any( + step.kind == ATTEMPT for step in spec.steps + ): + problems.append( + f"{spec.id}: a deny check needs a real attempt, not only a review" + ) + return problems + + +def validate_targets(targets: Targets) -> list[str]: + """Return release-input problems before a runner or network call exists.""" + problems: list[str] = [] + sha = re.compile(r"[0-9a-f]{40}\Z") + image = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" + ) + if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: + problems.append( + "repository and remote must be the fixed Atlas titan-iac origin" + ) + if ( + targets.reviewed_pr_number != 19 + or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" + ): + problems.append( + "reviewed PR identity must remain fixed to draft PR #19 and its existing branch" + ) + for label, value in ( + ("baseline commit", targets.baseline_commit), + ("remote main SHA", targets.remote_main_sha), + ("reviewed head SHA", targets.reviewed_head_sha), + ("build SHA", targets.build_sha), + ): + if not sha.fullmatch(value): + problems.append(f"{label} must be an exact lowercase 40-character SHA") + image_match = image.fullmatch(targets.agent_image) + if not image_match or image_match.group(1) != targets.build_sha: + problems.append("agent image tag/digest must bind the exact build SHA") + if targets.build_sha != targets.remote_main_sha: + problems.append("build SHA must equal the exact release main SHA") + if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): + problems.append("deployment revision must be a positive integer string") + if targets.node_count != 3 or targets.pool_replicas != 3: + problems.append("node and pool counts are fixed at exactly three") + if not targets.expect_telegram_sessions or not targets.chat_config_revision.strip(): + problems.append( + "Telegram continuity and its chat config revision are mandatory" + ) + if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: + problems.append("pool assignment evidence must include worker node and ordinal") + heads = dict(targets.dependency_heads) + if len(heads) != len(targets.dependency_heads) or set(heads) != set( + targets.dependency_pull_requests + ): + problems.append("every dependency PR must have one exact current head") + elif any(not sha.fullmatch(value) for value in heads.values()): + problems.append("dependency heads must be exact lowercase 40-character SHAs") + if ( + not isinstance(targets.max_evidence_age_seconds, int) + or targets.max_evidence_age_seconds <= 0 + ): + problems.append("maximum evidence age must be a positive integer") return problems def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: """Return the authenticated username of a vantage and any failure detail.""" - outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + outcome = runner.run( + ( + "kubectl", + "auth", + "whoami", + "-o", + projection("jsonpath={.status.userInfo.username}"), + ), + vantage, + ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - try: - payload = json.loads(outcome.stdout) - except ValueError as exc: - return ("", f"unparsable identity: {exc}") - username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") - if not isinstance(username, str) or not username: + username = outcome.stdout.strip() + if not username or any(character.isspace() for character in username): return ("", "identity response carried no username") return (username, "") -def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: +def _first_pod( + runner: Runner, operator: Vantage, namespace: str, selector: str +) -> tuple[str, str]: outcome = runner.run( ( "kubectl", @@ -119,14 +214,21 @@ def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + names = [ + line.strip().removeprefix("pod/") + for line in outcome.stdout.splitlines() + if line.strip() + ] if not names: return ("", f"no running pod matches {selector}") return (sorted(names)[0], "") def resolve_vantages( - runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None + runner: Runner, + targets: Targets, + kubeconfig: str | None = None, + context: str | None = None, ) -> tuple[dict[str, Vantage], list[VantageRecord]]: """Resolve every vantage the catalog names, recording what each one is.""" operator = operator_vantage(kubeconfig, context) @@ -145,23 +247,37 @@ def resolve_vantages( return (vantages, records) for name, selector, container, description in ( - (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SELF, + f"app={targets.agent_deployment}", + targets.agent_container, + "the Hermes agent itself", + ), ( SWITCHYARD, f"app={targets.switchyard_deployment}", targets.switchyard_container, "the Switchyard routing evidence log", ), - (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ( + NODE, + f"app={targets.node_daemonset}", + targets.node_container, + "a node hardening probe", + ), ): pod, failure = _first_pod(runner, operator, targets.namespace, selector) if not pod: records.append( - VantageRecord(name=name, description=description, available=False, detail=failure) + VantageRecord( + name=name, description=description, available=False, detail=failure + ) ) continue vantage = pod_vantage(targets.namespace, pod, container, operator) - identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + identity, identity_detail = ( + resolve_identity(runner, vantage) if name == SELF else ("", "") + ) records.append( VantageRecord( name=name, @@ -175,11 +291,21 @@ def resolve_vantages( chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" present = runner.run( - ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + ( + "kubectl", + "--namespace", + targets.namespace, + "get", + f"pod/{chat_pod}", + "-o", + "name", + ), operator, ) if present.ok and present.stdout.strip(): - vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + vantages[CHAT] = pod_vantage( + targets.namespace, chat_pod, targets.chat_container, operator + ) records.append( VantageRecord( name=CHAT, @@ -193,7 +319,9 @@ def resolve_vantages( name=CHAT, description=f"the chat surface ({targets.namespace}/{chat_pod})", available=False, - detail=present.error or present.combined[:200] or "chat tenant pod not found", + detail=present.error + or present.combined[:200] + or "chat tenant pod not found", ) ) return (vantages, records) @@ -206,10 +334,14 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: hermes = by_name.get(SELF) problems: list[str] = [] if operator is None or not operator.available: - problems.append("the operator vantage is unavailable; no evidence can be collected") + problems.append( + "the operator vantage is unavailable; no evidence can be collected" + ) return problems if hermes is None or not hermes.available: - problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + problems.append( + "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" + ) return problems if not hermes.identity: problems.append("the in-pod vantage did not report an identity") @@ -218,6 +350,10 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: "the operator and in-pod vantages authenticate as the same principal " f"({operator.identity}); run the harness with a separate operator kubeconfig" ) + for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): + record = by_name.get(name) + if record is None or not record.available: + problems.append(f"the required {label} vantage is unavailable") return problems @@ -231,7 +367,9 @@ def run_check( vantage = vantages.get(step.vantage) if vantage is None: outcome = Outcome( - argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + argv=step.argv, + vantage=step.vantage, + error=f"vantage {step.vantage} is unavailable", ) else: outcome = runner.run(step.argv, vantage, step.max_bytes) @@ -300,18 +438,27 @@ def build_report( harness_errors=problems, ) armed_ids = {result.spec.id for result in extra_results} - report.results = [ - *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), - *extra_results, - ] + runnable = [spec for spec in specs if spec.id not in armed_ids] + if problems: + report.results = [ + CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") + for spec in runnable + ] + report.results.extend(extra_results) + else: + report.results = [*run_catalog(runner, runnable, vantages), *extra_results] report.results.sort(key=lambda result: result.spec.id) if runner.remaining_seconds <= 0: - report.harness_errors.append("the run deadline expired before every check completed") + report.harness_errors.append( + "the run deadline expired before every check completed" + ) report.finished_at = utc_now() return report -def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: +def unavailable_report( + mode: str, started_at: str, reason: str, targets: Targets +) -> Report: """Return a NO_GO report for a run that could not start at all.""" return Report( mode=mode, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..9c3de171 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,21 @@ } ], "managed_modules": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -25,6 +40,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", @@ -35,6 +51,21 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -57,6 +88,7 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "scripts/ops", "scripts.render.dashboards_render_atlas", "services/mailu/scripts", "testing" @@ -118,7 +150,26 @@ ], "hygiene": { "max_lines": 500, + "legacy_line_exceptions": { + "scripts/tests/test_dashboards_render_atlas.py": { + "lines": 516, + "sha256": "afe4923391b157d03bcf0899208f77eeab79a2762da3575a365530b3074c5f25" + }, + "testing/tests/test_hermes_chat_quality.py": { + "lines": 2242, + "sha256": "71e9cafd14460b5132d07dd951514756d091009c23db7703db1575b05e34e75f" + }, + "testing/tests/test_hermes_cli_lanes.py": { + "lines": 1971, + "sha256": "992bf3788c33252e37a695f5307c31b1e8c0646e00a715e3e55f6901788abb62" + }, + "testing/tests/test_hermes_coordinator.py": { + "lines": 510, + "sha256": "290240cf391abc755eaa0abb12d247ad1718e7c9295a2092f349a7eaba4bfe55" + } + }, "line_limit_globs": [ + "scripts/ops/hermes_handoff_*.py", "testing/**/*.py", "ci/scripts/*.py", "ci/tests/**/*.py", @@ -167,7 +218,44 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "ci/scripts/semgrep_report.py", + "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", + "testing/quality_hygiene.py" + ], "tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -178,6 +266,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py" ] } diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..04355e51 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -46,18 +57,33 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] return [f"coverage xml missing: {xml_path.relative_to(root)}"] percentages = _load_percentages(xml_path, root) - minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + minimum = float(coverage_contract.get("minimum_percent", 95.0)) + branch_minimum = float(coverage_contract.get("minimum_branch_percent", 95.0)) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get("branch_tracked_files", ()) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + f"branch coverage below {branch_minimum:.1f}%: " + f"{relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +103,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py new file mode 100644 index 00000000..a0c63fa0 --- /dev/null +++ b/testing/quality_handoff_mutation.py @@ -0,0 +1,209 @@ +"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Mutation: + """One source mutation and the isolated probe that must kill it.""" + + name: str + path: str + old: str + new: str + probe: str + + +@dataclass(frozen=True) +class MutationReport: + """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" + + total: int + killed: int + issues: tuple[str, ...] + + @property + def score(self) -> float: + return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) + + +MUTATIONS = ( + Mutation( + "policy-allows-git-fetch", + "hermes_handoff_policy.py", + ' "for-each-ref",\n', + ' "fetch",\n "for-each-ref",\n', + "from hermes_handoff_policy import *\n" + "try: check_argv(('git','fetch','origin'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "redaction-truncates-before-screening", + "hermes_handoff_redaction.py", + " return bound(redact(text), max_bytes)\n", + " return bound(text, max_bytes)\n", + "from hermes_handoff_redaction import safe_text\n" + "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", + ), + Mutation( + "empty-report-go", + "hermes_handoff_model.py", + " if (\n" + " not self.results\n" + " or self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + " if (\n" + " self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", + ), + Mutation( + "generic-404-is-denial", + "hermes_handoff_rules.py", + "DENIAL_MARKERS = (\n", + 'DENIAL_MARKERS = (\n "404 not found",\n', + "from hermes_handoff_rules import looks_denied\n" + "from hermes_handoff_exec import Outcome\n" + "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", + ), + Mutation( + "truncated-evidence-reaches-evaluator", + "hermes_handoff_rules.py", + " if truncated:\n" + " return Evaluation(\n" + " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" + " )\n", + " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators\n" + "import hermes_handoff_rules as r\n" + "from hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" + "s=CheckSpec('x','x','x','probe')\n" + "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", + ), + Mutation( + "truncated-outcome-is-ok", + "hermes_handoff_exec.py", + " return self.error is None and self.returncode == 0 and not self.truncated\n", + " return self.error is None and self.returncode == 0\n", + "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", + ), + Mutation( + "nan-is-numeric", + "hermes_handoff_json_rules.py", + " or not math.isfinite(float(actual))\n", + " or False\n", + "import hermes_handoff_json_rules as j\n" + "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" + "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" + "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" + "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", + ), + Mutation( + "cleanup-ignores-branch-delete-failure", + "hermes_handoff_ephemeral.py", + ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', + ' if False and not deleted.ok:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" + "class R:\n" + " def __init__(self): self.i=0\n" + " def run(self,*_a,**_k):\n" + " self.i+=1\n" + " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", + ), + Mutation( + "cleanup-accepts-open-pr", + "hermes_handoff_ephemeral.py", + ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', + ' elif False:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" + "assert e._closed_pr_problems(p,q)\n", + ), +) + + +def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "PYTHONPATH": str(root), + } + return subprocess.run( + [sys.executable, "-c", probe], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + env=environment, + ) + + +def run_gate( + repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS +) -> MutationReport: + """Run baseline and mutated probes in isolated copies and return the score.""" + source_root = repo_root / "scripts" / "ops" + issues: list[str] = [] + killed = 0 + with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: + clean = Path(temporary) / "clean" + shutil.copytree(source_root, clean) + for mutation in mutations: + source = clean / mutation.path + original = source.read_text(encoding="utf-8") + if original.count(mutation.old) != 1: + issues.append(f"{mutation.name}: mutation anchor count is not one") + continue + baseline = _probe(clean, mutation.probe) + if baseline.returncode != 0: + issues.append(f"{mutation.name}: baseline probe failed") + continue + mutated_text = original.replace(mutation.old, mutation.new, 1) + try: + compile(mutated_text, str(source), "exec") + except SyntaxError: + issues.append(f"{mutation.name}: mutant is syntactically invalid") + continue + mutant = Path(temporary) / f"mutant-{mutation.name}" + shutil.copytree(clean, mutant) + (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") + if _probe(mutant, mutation.probe).returncode != 0: + killed += 1 + else: + issues.append(f"{mutation.name}: mutant survived") + return MutationReport(len(mutations), killed, tuple(issues)) + + +def main() -> int: + """Run the repository mutation contract and print a bounded score.""" + report = run_gate(Path(__file__).resolve().parents[1]) + print( + f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" + ) + for issue in report.issues: + print(issue) + return 0 if not report.issues and report.killed == report.total else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) # pragma: no cover diff --git a/testing/quality_hygiene.py b/testing/quality_hygiene.py index 5060ca02..6b68de3a 100644 --- a/testing/quality_hygiene.py +++ b/testing/quality_hygiene.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re from collections.abc import Iterable from pathlib import Path @@ -16,6 +17,18 @@ def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]: return sorted(matched) +def _is_unchanged_legacy( + config: dict[str, Any], root: Path, path: Path, line_count: int +) -> bool: + """Allow an over-cap legacy file only while its exact bytes stay unchanged.""" + relative = path.relative_to(root).as_posix() + record = config.get("legacy_line_exceptions", {}).get(relative) + if not isinstance(record, dict) or record.get("lines") != line_count: + return False + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return record.get("sha256") == digest + + def run_check(contract: dict[str, Any], root: Path) -> list[str]: """Return human-readable issues for naming and file-size rules.""" config = contract.get("hygiene", {}) @@ -24,8 +37,12 @@ def run_check(contract: dict[str, Any], root: Path) -> list[str]: for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: - issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})") + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): + issues.append( + f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})" + ) for rule in config.get("naming_rules", []): pattern = re.compile(rule["pattern"]) @@ -48,7 +65,9 @@ def count_files_over_line_limit(contract: dict[str, Any], root: Path) -> int: count = 0 for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): count += 1 return count diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py new file mode 100644 index 00000000..ecd64eb4 --- /dev/null +++ b/testing/tests/conftest.py @@ -0,0 +1,20 @@ +"""Hermetic runtime-path fixtures shared by repository tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_coordinator_gitea_token( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Prevent coordinator tests from observing the mounted production token.""" + if request.node.path.name == "test_hermes_coordinator.py": + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token") + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..f8a333b1 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,6 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) + monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {}) + monkeypatch.setattr( + module, + "_configured_models", + lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"], + ) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py index 87c8adbe..4f89db35 100644 --- a/testing/tests/test_hermes_handoff_acceptance.py +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -18,7 +18,9 @@ harness_run = load_handoff_module("hermes_handoff_run") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) class ScriptedCluster: @@ -44,10 +46,37 @@ class ScriptedCluster: def whoami(username: str): - return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + return completed(stdout=username) + + +REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" +SHA = "1" * 40 +VALID_ARGS = [ + "--remote-main-sha", + SHA, + "--reviewed-head-sha", + REVIEWED, + "--agent-image", + f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", + "--build-sha", + SHA, + "--deployment-revision", + "1", + "--chat-config-revision", + "chat-rev", + *[ + item + for number in catalog.Targets().dependency_pull_requests + for item in ("--dependency-head", f"{number}={SHA}") + ], +] HEALTHY_CLUSTER = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("kubernetes-admin")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), @@ -60,10 +89,18 @@ def run_cli(monkeypatch, argv, rules): clock = FakeClock() def build_runner(**kwargs): - return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + return exec_module.Runner( + clock=clock, + spawn=cluster, + environ={}, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) monkeypatch.setattr(acceptance, "Runner", build_runner) - return (acceptance.main(argv), cluster) + return (acceptance.main([*VALID_ARGS, *argv]), cluster) def read_report(path): @@ -109,7 +146,15 @@ def test_defaults_fall_back_to_the_shipped_targets() -> None: assert targets.repo == catalog.Targets.repo -def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: +@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) +def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: + with pytest.raises(Exception): + acceptance.dependency_head(value) + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER @@ -124,13 +169,15 @@ def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_p writes = [ call for call in cluster.calls - if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + if {"patch", "create", "delete", "apply", "replace"} & set(call) + and "can-i" not in call ] - assert writes - assert all("--dry-run=server" in call for call in writes) + assert writes == [] -def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: +def test_mutating_checks_report_not_run_until_the_mode_is_armed( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) @@ -163,7 +210,10 @@ def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( assert report["decision"] == model.NO_GO assert report["counts"][model.PASS] == 0 assert sum(report["counts"].values()) == len(report["checks"]) - assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + assert any( + "in-pod Hermes vantage is unavailable" in error + for error in report["harness_errors"] + ) def test_partial_failure_is_reported_per_check_rather_than_aborting( @@ -186,14 +236,25 @@ def test_partial_failure_is_reported_per_check_rather_than_aborting( assert report["blocking"] -def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" rules = [ - ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), - ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ( + "get pods --selector app=hermes-agent", + completed(stdout="pod/hermes-agent-1\n"), + ), ("get pods --selector", completed(stdout="pod/other-1\n")), - ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ( + "get pod/hermes-chat-tenant-0", + completed(stdout="pod/hermes-chat-tenant-0\n"), + ), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) @@ -202,16 +263,18 @@ def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_ assert any("same principal" in error for error in report["harness_errors"]) -def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: +def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, _ = run_cli( - monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + monkeypatch, + ["--output", str(output), "--json-only", "--deadline", "0"], + HEALTHY_CLUSTER, ) report = read_report(output) - assert status == acceptance.EXIT_NO_GO + assert status == acceptance.EXIT_USAGE assert report["counts"][model.NOT_RUN] > 0 - assert any("deadline expired" in error for error in report["harness_errors"]) + assert any("invalid execution bound" in error for error in report["harness_errors"]) def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( @@ -239,7 +302,9 @@ def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( assert not any("push" in call for call in cluster.calls) -def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: +def test_arming_against_a_protected_push_target_is_refused( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, @@ -261,17 +326,24 @@ def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path assert not any("push" in call for call in cluster.calls) -def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( +def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) rules = [ ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), ("git push", completed()), - ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ( + "POST /api/v1/repos/atlas/titan-iac/pulls", + completed(stdout=json.dumps({"number": 9})), + ), ("PATCH", completed()), ("DELETE", completed()), - ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + ( + "GET /api/v1/repos/atlas/titan-iac/pulls/9", + completed(stdout='{"state": "closed"}'), + ), *HEALTHY_CLUSTER, ] status, cluster = run_cli( @@ -289,20 +361,14 @@ def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( rules, ) report = read_report(output) - ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} - assert report["mode"] == policy.ARMED - assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven - assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS - assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS - assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS - assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS - + assert status == acceptance.EXIT_NO_GO + assert any( + "read-only acceptance did not GO" in error for error in report["harness_errors"] + ) pushes = [call for call in cluster.calls if "push" in call] - assert len(pushes) == 1 - assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") - assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator - assert any("DELETE" in call for call in cluster.calls) + assert pushes == [] + assert not any("DELETE" in call for call in cluster.calls) def test_the_report_is_written_to_stdout_when_no_output_path_is_given( @@ -327,10 +393,14 @@ def test_the_human_summary_accompanies_the_json_unless_suppressed( assert captured.out == "" -def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: +def test_a_report_that_fails_its_own_screening_is_replaced_not_published( + tmp_path, +) -> None: """The last barrier: never write an artifact nobody re-read.""" output = tmp_path / "report.json" - arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + arguments = acceptance.build_parser().parse_args( + ["--output", str(output), "--json-only"] + ) report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] @@ -348,20 +418,28 @@ def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_pat assert "failed its own credential screening" in payload["harness_errors"][0] -def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: +def test_a_screening_failure_forces_no_go_from_the_entry_point( + monkeypatch, tmp_path +) -> None: """The exit status must follow the refusal to publish, not the check tally.""" output = tmp_path / "report.json" - monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr( + acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] + ) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) assert status == acceptance.EXIT_NO_GO assert read_report(output)["decision"] == model.NO_GO -def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: +def test_no_recorded_command_output_carries_a_credential_shape( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") run_cli( @@ -380,16 +458,3 @@ def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> Non acceptance.main([flag]) assert "Read-only by default" in capsys.readouterr().out - - -def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: - """Exercise the success exit path without pretending the cluster is clean.""" - output = tmp_path / "report.json" - monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) - monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) - monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) - - assert status == acceptance.EXIT_GO - assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_acceptance_armed.py b/testing/tests/test_hermes_handoff_acceptance_armed.py new file mode 100644 index 00000000..ded33bb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance_armed.py @@ -0,0 +1,120 @@ +"""Static and armed-transition regressions for the handoff CLI.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_acceptance import ( + HEALTHY_CLUSTER, + REVIEWED, + VALID_ARGS, + acceptance, + harness_run, + model, + policy, + read_report, + run_cli, +) + + +def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None: + base = model.CheckSpec( + id="release.pass", title="pass", group="release", rule="allowed" + ) + placeholder = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="pending", + group="ephemeral", + rule="not_armed", + mandatory=False, + scope=model.EPHEMERAL, + ) + report = model.Report( + mode=policy.ARMED, + started_at="2026-08-17T00:00:00Z", + results=[ + model.CheckResult(base, model.PASS), + model.CheckResult(placeholder, model.NOT_RUN), + ], + ) + armed_spec = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="clean", + group="ephemeral", + rule="armed", + scope=model.EPHEMERAL, + ) + armed = model.CheckResult(armed_spec, model.PASS) + monkeypatch.setattr( + acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, []) + ) + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + monkeypatch.setattr(acceptance, "build_report", lambda *_args: report) + monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed]) + monkeypatch.setattr(acceptance, "emit", lambda *_args: []) + status = acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + assert status == acceptance.EXIT_GO + assert ( + next( + result + for result in report.results + if result.spec.id.startswith("ephemeral") + ).status + == model.PASS + ) + monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"]) + assert ( + acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817b", + ] + ) + == acceptance.EXIT_NO_GO + ) + + +def test_armed_results_are_bound_to_operator_and_exact_reviewed_head( + monkeypatch, +) -> None: + arguments = acceptance.build_parser().parse_args( + [ + *VALID_ARGS, + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + calls = [] + monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args)) + monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args]) + operator = object() + result = acceptance._armed_results(arguments, object(), {"operator": operator}) + assert calls[0][0].expected_head == REVIEWED + assert result[0][1] is operator diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py index 8f5e2de6..21e8031a 100644 --- a/testing/tests/test_hermes_handoff_catalog.py +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -23,8 +23,12 @@ BY_ID = {check.id: check for check in SPECS} VANTAGES = { catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), - catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), - catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.SWITCHYARD: exec_module.pod_vantage( + "hermes", "hermes-switchyard-1", "switchyard" + ), + catalog.NODE: exec_module.pod_vantage( + "hermes", "hermes-node-ssh-access-1", "key-reconciler" + ), catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), } @@ -42,32 +46,41 @@ def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> No policy.check_argv(addressed, policy.READ_ONLY) -def test_no_step_mutates_outside_a_server_dry_run() -> None: - """`auth can-i create ...` is a read; the subcommand is what decides.""" +def test_no_step_constructs_a_mutation_or_dry_run() -> None: + """Authorization review may name a verb; no executed subcommand mutates.""" for check in SPECS: for step in check.steps: subcommand = next(policy.positionals(step.argv), "") - if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: - assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + assert subcommand not in {"apply", "create", "delete", "patch", "replace"} + assert not any(argument.startswith("--dry-run") for argument in step.argv) -def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: +def test_no_catalog_step_constructs_impersonation() -> None: operator_steps = [ - step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + step + for check in SPECS + for step in check.steps + if step.vantage == catalog.OPERATOR ] assert operator_steps - assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps + ) - impersonation = BY_ID["access.impersonation-is-denied"] - attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] - assert any("--as" in step.argv for step in attempts) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) + for check in SPECS + for step in check.steps + ) -def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: - deny_checks = [check for check in SPECS if check.rule == "denied"] - assert len(deny_checks) >= 6 +def test_credential_or_mutation_denials_are_review_only() -> None: + deny_checks = [check for check in SPECS if check.rule == "review_denied"] + assert len(deny_checks) >= 5 for check in deny_checks: - assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + assert check.steps and all(step.kind == model.REVIEW for step in check.steps), ( + check.id + ) def test_both_vantages_are_represented_across_the_catalog() -> None: @@ -104,7 +117,9 @@ def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: for number in TARGETS.dependency_pull_requests: check = BY_ID[f"baseline.dependency-pr-{number}-merged"] assert check.mandatory - assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + assert check.expect["fields"]["merged"] is True + assert check.expect["fields"]["base.ref"] == "main" + assert "head.sha" in check.expect["fields"] def test_the_baseline_commit_gate_names_the_configured_commit() -> None: @@ -113,25 +128,12 @@ def test_the_baseline_commit_gate_names_the_configured_commit() -> None: assert "--is-ancestor" in check.steps[0].argv -def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: - """A names_present over an empty expectation would pass on any workload.""" - assert "pool.assignment-safety-knobs-are-configured" not in BY_ID - assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID - assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID - - configured = harness_run.build_catalog( - catalog.Targets( - now=NOW, - pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), - chat_config_revision="20260816-telegram-topics", - expect_telegram_sessions=True, - ) - ) - identifiers = {check.id for check in configured} - assert "pool.assignment-safety-knobs-are-configured" in identifiers - assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers - assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers - assert harness_run.validate_catalog(configured) == [] +def test_pool_assignment_and_chat_continuity_are_unconditional() -> None: + assert "pool.assignment-safety-knobs-are-configured" in BY_ID + assert "pool.assignment-fieldrefs-are-bound" in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID + assert TARGETS.node_count == TARGETS.pool_replicas == 3 def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: @@ -144,7 +146,10 @@ def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: def test_freshness_gates_carry_the_run_clock() -> None: - for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + for identifier in ( + "identity.codex-evidence-is-fresh", + "identity.claude-evidence-is-fresh", + ): assert BY_ID[identifier].expect["now"] == NOW routing = BY_ID["routing.provider-effort-and-fallback-evidence"] assert routing.expect["now"] == NOW @@ -170,18 +175,14 @@ def test_check_identifiers_are_unique_and_grouped() -> None: "access.workload-mutation-is-denied", "access.pod-exec-is-denied", "access.required-reads-are-allowed", - "access.pod-logs-are-allowed", "access.flux-and-helm-status-are-allowed", "access.namespace-view-agrees-across-vantages", - "nodes.account-password-is-locked", "nodes.hardening-covers-every-node", "build.builder-service-account-is-tokenless", "build.harbor-immutability-rule-is-applied", - "build.deployed-image-is-digest-pinned", + "release.exact-lineage-is-running", "reliability.finalization-and-replay-patches-are-deployed", - "pool.three-workers-are-ready", - "pool.workers-occupy-distinct-nodes", - "pool.workers-own-distinct-volumes", + "pool.three-workers-have-distinct-nodes-and-volumes", "pool.coordinator-retains-sole-state-ownership", "surfaces.kanban-activity-is-visible", "surfaces.chat-telegram-topic-state-is-durable", diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py index 09cc5d63..9f54601a 100644 --- a/testing/tests/test_hermes_handoff_ephemeral.py +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -1,9 +1,10 @@ -"""Contracts for the separately armed, self-cleaning mutation mode.""" +"""Contracts for the strictly bounded, self-cleaning armed mode.""" from __future__ import annotations import json import subprocess +from pathlib import Path import pytest @@ -18,254 +19,319 @@ exec_module = load_handoff_module("hermes_handoff_exec") policy = load_handoff_module("hermes_handoff_policy") model = load_handoff_module("hermes_handoff_model") -REPO = "atlas/titan-iac" TOKEN = "acceptance-20260817a" +HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac" +MAIN = "1" * 40 VANTAGE = exec_module.operator_vantage() -def request(**overrides) -> object: +def request(**overrides): fields = { - "repo": REPO, - "remote": "origin", + "repo": policy.EXPECTED_REPO, + "remote": policy.EXPECTED_REMOTE, "token": TOKEN, "confirmation": ephemeral.CONFIRMATION, - "base": "main", + "base": policy.EXPECTED_BASE, + "expected_head": HEAD, + "expected_base_sha": MAIN, } fields.update(overrides) return ephemeral.ArmRequest(**fields) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) -def armed_runner(results: list) -> tuple: +def armed_runner(results: list, mode=policy.ARMED): spawn = FakeSpawn(results) runner = exec_module.Runner( - mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + mode=mode, + clock=FakeClock(), + spawn=spawn, + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), ) - return (runner, spawn) + return runner, spawn def by_id(results: list) -> dict: return {result.spec.id: result for result in results} -def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: - assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" - - -@pytest.mark.parametrize( - "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] -) -def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: - with pytest.raises(ephemeral.ArmingError): - ephemeral.assert_push_target_allowed(ref) - - -def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: - ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") - - -@pytest.mark.parametrize( - ("overrides", "fragment"), - [ - ({"confirmation": "yes please"}, "confirmation phrase"), - ({"confirmation": ""}, "confirmation phrase"), - ({"repo": "atlas/other"}, "is not the expected"), - ({"token": "short"}, "ephemeral token"), - ({"token": "Has-Capitals-Here"}, "ephemeral token"), - ({"token": ""}, "ephemeral token"), - ({"base": ""}, "base branch is required"), - ], -) -def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: - with pytest.raises(ephemeral.ArmingError) as caught: - ephemeral.preflight(request(**overrides), REPO) - assert fragment in str(caught.value) - - -def test_preflight_accepts_a_correctly_armed_request() -> None: - ephemeral.preflight(request(), REPO) - - -@pytest.mark.parametrize( - ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] -) -def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: - assert ephemeral.normalise_branch(raw) == expected - - -def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: - """A guard checked once is a guard nobody notices regressing.""" - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - guard = results["ephemeral.protected-branch-refusal"] - - assert guard.status == model.PASS - assert guard.evidence["accepted"] == [] - assert "main" in guard.evidence["refused"] - - -def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: - """If the guard ever stops refusing, the armed run says so instead of pushing.""" - monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ - "ephemeral.protected-branch-refusal" - ] - - assert guard.status == model.FAIL - assert "main" in guard.evidence["accepted"] - assert "the guard accepted" in guard.reason - - -def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: - runner, spawn = armed_runner( - [ - completed(stdout=""), # ls-remote: ref unused - completed(stdout=""), # push - completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft - completed(stdout=""), # close - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - completed(stdout=json.dumps({"number": 42, "state": "closed"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.PASS - assert results["ephemeral.draft-pull-request"].status == model.PASS - assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 - assert results["ephemeral.cleanup-verified"].status == model.PASS - assert all(result.spec.mandatory for result in results.values()) - assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) - - pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] - assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" - assert "--force" not in pushed[0]["argv"] - - titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] - assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") - - -def test_an_existing_ref_stops_the_run_before_it_writes() -> None: - runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert "already exists" in results["ephemeral.feature-branch-push"].reason - assert "ephemeral.draft-pull-request" not in results - assert len(spawn.calls) == 1 - - -def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: - runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN - - -def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stderr="denied", returncode=1), - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN - assert results["ephemeral.cleanup-verified"].status == model.PASS - - -def test_a_draft_response_without_a_number_is_a_failure() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout="{}"), - completed(stdout=""), - completed(stdout=""), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.draft-pull-request"].status == model.FAIL - - -def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stderr="cannot close", returncode=1), - completed(stderr="cannot delete", returncode=1), - completed(stdout="abc123\trefs/heads/x\n"), - completed(stdout=json.dumps({"number": 7, "state": "open"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - cleanup = results["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not be closed" in cleanup.reason - assert "still exists" in cleanup.reason - assert "still open" in cleanup.reason - - -def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stdout=""), - completed(stdout=""), - subprocess.TimeoutExpired(cmd="git", timeout=1.0), - completed(stderr="gone", returncode=1), - ] - ) - - cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not verify removal" in cleanup.reason - assert "could not confirm pull request #7 is closed" in cleanup.reason - - -def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: - runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) - identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} - - assert identifiers <= { - "ephemeral.protected-branch-refusal", - "ephemeral.feature-branch-push", - "ephemeral.draft-pull-request", - "ephemeral.cleanup-verified", +def pr_payload(state="open", number=42): + return { + "number": number, + "state": state, + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": request().ref, "sha": HEAD}, } -def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: - """The mode, not the caller, is what opens the mutations.""" - spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) - runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) +def linked_worktree(tmp_path: Path) -> Path: + root = tmp_path / "worktree" + common = tmp_path / "repo.git" + gitdir = common / "worktrees" / "acceptance" + gitdir.mkdir(parents=True) + (root).mkdir() + (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + (gitdir / "commondir").write_text("../..\n", encoding="utf-8") + (gitdir / "HEAD").write_text( + f"ref: refs/heads/{ephemeral.EXPECTED_HEAD_REF}\n", encoding="utf-8" + ) + ref = common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF + ref.parent.mkdir(parents=True) + ref.write_text(f"{HEAD}\n", encoding="utf-8") + remote_ref = common / "refs" / "remotes" / "origin" / "main" + remote_ref.parent.mkdir(parents=True) + remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") + (common / "config").write_text( + f'[remote "origin"]\n\turl = {ephemeral.EXPECTED_REMOTE_URL}\n', + encoding="utf-8", + ) + return root + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""] +) +def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + ephemeral.assert_push_target_allowed(request().ref) + + +@pytest.mark.parametrize( + "overrides", + [ + {"confirmation": "wrong"}, + {"repo": "atlas/other"}, + {"remote": "upstream"}, + {"base": "develop"}, + {"token": "short"}, + {"expected_head": "bad"}, + {"expected_base_sha": "bad"}, + ], +) +def test_preflight_rejects_bad_inputs_before_any_runner_exists( + tmp_path: Path, overrides: dict +) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.preflight( + request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path) + ) + + +def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None: + ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path)) + root = linked_worktree(tmp_path / "wrong") + (Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text( + "ref: refs/heads/other\n", encoding="utf-8" + ) + with pytest.raises(ephemeral.ArmingError, match="existing PR #19"): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + + +def test_worktree_attestation_rejects_each_malformed_local_source( + tmp_path: Path, +) -> None: + roots = [] + empty = tmp_path / "empty" + empty.mkdir() + roots.append(empty) + bad_marker = linked_worktree(tmp_path / "marker") + (bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8") + roots.append(bad_marker) + wrong_sha = linked_worktree(tmp_path / "sha") + gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF).write_text("0" * 40) + roots.append(wrong_sha) + wrong_main = linked_worktree(tmp_path / "main") + gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40) + roots.append(wrong_main) + bad_config = linked_worktree(tmp_path / "config") + gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text("[broken\n", encoding="utf-8") + roots.append(bad_config) + wrong_remote = linked_worktree(tmp_path / "remote") + gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n') + roots.append(wrong_remote) + for root in roots: + with pytest.raises((ephemeral.ArmingError, FileNotFoundError)): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + oversized = tmp_path / "oversized" + oversized.write_text("x" * 5, encoding="utf-8") + with pytest.raises(ephemeral.ArmingError): + ephemeral._bounded_read(oversized, 4) + + +def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, spawn = armed_runner([]) results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - push = results["ephemeral.feature-branch-push"] + assert results["ephemeral.protected-branch-refusal"].status == model.FAIL + assert all( + result.status in {model.FAIL, model.NOT_RUN} for result in results.values() + ) + assert spawn.calls == [] - assert push.status == model.FAIL - assert "policy" in push.reason - assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) + +def test_full_armed_run_validates_and_removes_exact_artifacts() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert all(result.status == model.PASS for result in results.values()) + push = next(call for call in spawn.calls if call["argv"][1] == "push") + assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert all(result.spec.mandatory for result in results.values()) + + +def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None: + for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)): + runner, spawn = armed_runner([first]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any( + call["argv"][1] in {"push", "POST", "PATCH", "DELETE"} + for call in spawn.calls + ) + + +def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(returncode=1), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None: + assert ephemeral._pr_problems([], request()) == ["pull request is not an object"] + malformed = pr_payload() + malformed.update({"number": True, "draft": "yes", "base": None}) + malformed["head"] = {"ref": "other", "sha": "bad"} + problems = ephemeral._pr_problems(malformed, request()) + assert {"number is malformed", "draft does not match", "base is malformed"} <= set( + problems + ) + assert ephemeral._discover(exec_module.Outcome((), "x", returncode=1), request())[1] + for body in ( + "not-json", + "{}", + "[null]", + json.dumps([{"head": {"ref": "other"}, "base": {}}]), + ): + numbers, error = ephemeral._discover( + exec_module.Outcome((), "x", 0, body), request() + ) + assert numbers == [] and error + invalid = pr_payload() + invalid["draft"] = False + assert ephemeral._discover( + exec_module.Outcome((), "x", 0, json.dumps([invalid])), request() + )[1] + assert ( + "discovered 0" + in ephemeral._discover(exec_module.Outcome((), "x", 0, "[]"), request())[1] + ) + + +@pytest.mark.parametrize( + "delete_result,remaining,closed", + [ + (completed(returncode=1), completed(), pr_payload(state="closed")), + (completed(), completed(returncode=128), pr_payload(state="closed")), + (completed(), completed(), {}), + (completed(), completed(), pr_payload(state="open")), + ], +) +def test_cleanup_fails_closed_on_partial_or_malformed_evidence( + delete_result, remaining, closed +) -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + delete_result, + remaining, + completed(stdout=json.dumps(closed)), + ] + ) + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.cleanup-verified" + ] + assert cleanup.status == model.FAIL + + +def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> ( + None +): + api = "/api/v1/repos/atlas/titan-iac" + for scripted in ( + [completed(returncode=1), completed(), completed(), completed(returncode=1)], + [completed(stdout="bad"), completed(), completed(), completed(stdout="bad")], + [ + completed(stdout=json.dumps(pr_payload())), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ], + ): + runner, _ = armed_runner(scripted) + result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42]) + assert result.status == model.FAIL + + +def test_read_only_runner_cannot_reach_push() -> None: + runner, spawn = armed_runner( + [completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()], + mode=policy.READ_ONLY, + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any(call["argv"][1] == "push" for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py index 2e340c7b..4de16c7e 100644 --- a/testing/tests/test_hermes_handoff_evaluators.py +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -2,14 +2,25 @@ from __future__ import annotations -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") REVIEW = model.REVIEW ATTEMPT = model.ATTEMPT -FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) +FORBIDDEN = outcome( + stderr="Error from server (Forbidden): pods is forbidden", returncode=1 +) def deny_spec() -> object: @@ -24,21 +35,27 @@ def evaluate(built, outcomes): def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.PASS assert evaluation.evidence == {"review": "no", "attempt": "refused"} def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.FAIL assert "reported yes" in evaluation.reason def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} + ) assert evaluation.status == model.FAIL assert "must be refused" in evaluation.reason @@ -46,7 +63,9 @@ def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> No def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: broken = outcome(stderr="connection refused", returncode=7) - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} + ) assert evaluation.status == model.NOT_RUN assert "recognisable refusal" in evaluation.reason @@ -62,15 +81,22 @@ def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: assert "require a real attempt" in evaluation.reason -def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( + None +): built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) - assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + assert ( + evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status + == model.PASS + ) denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) assert denied.status == model.FAIL and "reported no" in denied.reason - failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + failed = evaluate( + built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} + ) assert failed.status == model.FAIL and "step read failed" in failed.reason @@ -84,7 +110,9 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 - leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + leaked = evaluate( + built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} + ) assert leaked.status == model.FAIL assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] @@ -92,10 +120,16 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: assert unreadable.status == model.NOT_RUN -def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( + None +): built = spec( "names_present", - {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + { + "step": "init", + "names": ("patch-web-session-activity",), + "contains": ("@sha256:",), + }, steps=(step("init"),), ) @@ -110,14 +144,18 @@ def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty( def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: - single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + single = spec( + "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) + ) assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" several = spec( - "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + "stdout_matches", + {"step": "s", "equals": ["600 root:root", "absent"]}, + steps=(step("s"),), ) assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS @@ -126,7 +164,9 @@ def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: def test_distinct_count_separates_replica_count_from_actual_spread() -> None: built = spec( - "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + "distinct_count", + {"step": "n", "minimum": 3, "total_equals": 3}, + steps=(step("n"),), ) spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) @@ -135,7 +175,11 @@ def test_distinct_count_separates_replica_count_from_actual_spread() -> None: stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) assert stacked.status == model.FAIL and "distinct values" in stacked.reason - extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + extra = spec( + "distinct_count", + {"step": "n", "minimum": 1, "total_equals": 2}, + steps=(step("n"),), + ) over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) assert over.status == model.FAIL and "expected exactly 2" in over.reason @@ -152,16 +196,22 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: agree = evaluate( built, - {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + { + "operator": outcome(stdout="ns/a\nns/b\n"), + "self": outcome(stdout="ns/b\nns/a\n"), + }, ) assert agree.status == model.PASS differ = evaluate( - built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, ) assert differ.status == model.FAIL and "disagree" in differ.reason - empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + empty = evaluate( + built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} + ) assert empty.status == model.NOT_RUN broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) @@ -169,7 +219,9 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: - assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + assert ( + evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + ) def test_lines_match_catches_a_blank_leading_column() -> None: @@ -186,7 +238,10 @@ def test_lines_match_catches_a_blank_leading_column() -> None: failed = evaluate(built, {"s": outcome(stdout=blank)}) assert failed.status == model.FAIL and failed.evidence["offenders"] - assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert ( + evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status + == model.NOT_RUN + ) assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN @@ -198,4 +253,205 @@ def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: assert evaluation.status == model.NOT_RUN assert evaluation.evidence == {"armed": False} assert evaluation.reason == "ephemeral mutation mode is not armed" - assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + assert ( + evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + ) + + +def test_truncation_fails_closed_before_every_registered_evaluator() -> None: + """No evaluator may reinterpret a clipped required capture as PASS.""" + clipped = outcome(stdout="apparently good", truncated=True) + for name in evaluators.EVALUATORS: + built = spec(name, {"step": "s"}, steps=(step("s"),)) + evaluation = evaluate(built, {"s": clipped}) + assert evaluation.status == model.NOT_RUN, name + assert "truncated" in evaluation.reason, name + + +def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: + for name in evaluators.EVALUATORS: + if name == "not_armed": + continue + built = spec(name, {"step": "s"}, steps=(step("s"),)) + assert evaluate(built, {}).status == model.NOT_RUN, name + + +def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: + built = spec( + "pool_topology", + { + "state_step": "state", + "worker_step": "workers", + "replicas": 3, + "name": "worker", + }, + steps=(step("state"), step("workers")), + ) + state = outcome(stdout="3\t3\t3\tfalse") + workers = outcome( + stdout=( + "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" + "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" + "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" + ) + ) + assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS + stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) + assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL + assert ( + evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} + ).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} + ).status + == model.FAIL + ) + malformed = outcome(stdout="worker-0\tnode-a\n") + assert ( + evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN + ) + bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) + assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL + duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) + assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL + wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) + assert ( + evaluate(built, {"state": state, "workers": wrong_ordinals}).status + == model.FAIL + ) + + +MAIN = "1" * 40 +HEAD = "2" * 40 +BUILD = MAIN +DIGEST = "sha256:" + "4" * 64 +IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" + + +def lineage_spec(): + keys = ( + "remote-main", + "reviewed-pr", + "build-source", + "deployment", + "pods", + "replicasets", + "flux", + ) + return spec( + "release_lineage", + { + "main_sha": MAIN, + "head_sha": HEAD, + "head_ref": "feature/hermes-full-handoff-acceptance", + "pr_number": 19, + "image": IMAGE, + "build_sha": BUILD, + "deployment_revision": "7", + }, + steps=tuple(step(key) for key in keys), + ) + + +def lineage_outcomes(): + pull = { + "number": 19, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, + } + return { + "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), + "reviewed-pr": outcome(stdout=json.dumps(pull)), + "build-source": outcome(stdout=BUILD), + "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), + "replicasets": outcome( + stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" + ), + "pods": outcome( + stdout=( + f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" + f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" + ) + ), + "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), + } + + +def test_release_lineage_atomically_binds_every_release_identity() -> None: + assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS + + +@pytest.mark.parametrize( + ("key", "bad"), + [ + ("remote-main", ""), + ("reviewed-pr", "{}"), + ("build-source", HEAD), + ("deployment", "5\t4\t2\t2\t7\tbad"), + ("replicasets", "malformed"), + ("pods", "malformed"), + ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), + ], +) +def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( + key, bad +) -> None: + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=bad) + assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} + + +def test_release_lineage_rejects_structural_partial_failure_variants() -> None: + variants = [] + malformed_expect = lineage_spec() + malformed_expect = model.CheckSpec( + **{ + **malformed_expect.__dict__, + "expect": {**malformed_expect.expect, "main_sha": "bad"}, + } + ) + variants.append((malformed_expect, lineage_outcomes())) + malformed_image = lineage_spec() + malformed_image = model.CheckSpec( + **{ + **malformed_image.__dict__, + "expect": { + **malformed_image.expect, + "image": f"registry.example/hermes@{DIGEST}", + }, + } + ) + variants.append((malformed_image, lineage_outcomes())) + for key, value in ( + ("reviewed-pr", "[]"), + ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), + ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), + ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), + ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), + ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), + ( + "pods", + f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ( + "pods", + f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), + ): + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=value) + variants.append((lineage_spec(), outcomes)) + for built, outcomes in variants: + assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN} diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py index cd8fc7d2..c192448d 100644 --- a/testing/tests/test_hermes_handoff_exec.py +++ b/testing/tests/test_hermes_handoff_exec.py @@ -2,7 +2,11 @@ from __future__ import annotations +import hashlib import subprocess +import time + +import pytest from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -15,35 +19,48 @@ policy = load_handoff_module("hermes_handoff_policy") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def build(results: list, **kwargs) -> tuple: clock = FakeClock() spawn = FakeSpawn(results) - runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + runner = runner_module.Runner( + clock=clock, + spawn=spawn, + environ={}, + attestor=lambda command, _environment: runner_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) return (runner, spawn, clock) OPERATOR = runner_module.operator_vantage() +SAFE_GET = ("kubectl", "get", "pods", "-o", "name") def test_a_successful_command_is_captured_and_screened() -> None: runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ok and outcome.ran assert "[redacted]" in outcome.stdout assert outcome.returncode == 0 - assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert spawn.calls[0]["argv"] == list(SAFE_GET) assert runner.commands_run == 1 def test_a_failing_command_is_not_ok_but_still_ran() -> None: - runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + runner, _, _ = build( + [completed(stderr="Error from server (Forbidden)", returncode=1)] + ) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ran and not outcome.ok assert "Forbidden" in outcome.combined @@ -63,7 +80,7 @@ def test_a_policy_violation_never_reaches_a_subprocess() -> None: def test_a_timeout_is_recorded_rather_than_raised() -> None: runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error == runner_module.TIMEOUT_ERROR assert not outcome.ran @@ -72,7 +89,7 @@ def test_a_timeout_is_recorded_rather_than_raised() -> None: def test_a_missing_binary_is_recorded_rather_than_raised() -> None: runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error and outcome.error.startswith("spawn:") @@ -80,9 +97,9 @@ def test_a_missing_binary_is_recorded_rather_than_raised() -> None: def test_the_run_deadline_stops_further_probes_without_spawning() -> None: runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) - assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + assert runner.run(SAFE_GET, OPERATOR).ok clock.advance(11.0) - second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR) assert second.error == runner_module.DEADLINE_ERROR assert len(spawn.calls) == 1 @@ -90,29 +107,42 @@ def test_the_run_deadline_stops_further_probes_without_spawning() -> None: def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: - runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + runner, spawn, clock = build( + [completed()], command_timeout=30.0, deadline_seconds=10.0 + ) clock.advance(6.0) - runner.run(("kubectl", "get", "pods"), OPERATOR) + runner.run(SAFE_GET, OPERATOR) assert spawn.calls[0]["timeout"] == 4.0 def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: - runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + runner, _, _ = build( + [completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50 + ) - small = runner.run(("kubectl", "get", "pods"), OPERATOR) - large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + small = runner.run(SAFE_GET, OPERATOR) + large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096) assert small.truncated assert not large.truncated def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: - source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + source = { + "PATH": "/bin", + "KUBECONFIG": "/tmp/kc", + "ANTHROPIC_API_KEY": "leak", + "HOME": "/root", + } environment = runner_module.build_environment(OPERATOR, source) - assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert environment["PATH"] == runner_module.SAFE_PATH + assert environment["KUBECONFIG"] == "/tmp/kc" + assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev" + assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "HOME" not in environment assert "ANTHROPIC_API_KEY" not in environment @@ -127,7 +157,11 @@ def test_a_vantage_can_override_the_environment_it_needs() -> None: def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: vantage = runner_module.operator_vantage(context="atlas-operator") - assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ( + "kubectl", + "--context", + "atlas-operator", + ) assert vantage.wrap(("git", "status")) == ("git", "status") assert vantage.wrap(()) == () @@ -136,11 +170,18 @@ def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> Non operator = runner_module.operator_vantage(context="atlas-operator") vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) - wrapped = vantage.wrap(("kubectl", "get", "pods")) + wrapped = vantage.wrap(SAFE_GET) assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") assert wrapped[3] == "exec" - assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert wrapped[-6:] == ( + "--", + "/usr/local/bin/kubectl", + "get", + "pods", + "-o", + "name", + ) assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" policy.check_argv(wrapped) @@ -168,6 +209,155 @@ def test_outcome_serialisation_screens_the_command_line() -> None: def test_combined_output_joins_only_the_streams_that_carry_text() -> None: - assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert ( + runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined + == "a\nb" + ) assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" assert runner_module.Outcome(argv=(), vantage="x").combined == "" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"command_timeout": float("nan")}, + {"command_timeout": float("inf")}, + {"command_timeout": 0}, + {"deadline_seconds": -1}, + {"max_bytes": -1}, + {"max_bytes": 10**100}, + {"concurrency": 0}, + {"concurrency": 17}, + ], +) +def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( + kwargs, +) -> None: + with pytest.raises(ValueError): + runner_module.Runner(**kwargs) + + +def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None: + for value in (True, "1", None): + with pytest.raises(ValueError, match="numeric"): + runner_module.Runner(command_timeout=value) + + +def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: + runner, spawn, _ = build([]) + commands = [ + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "get", "secret/x", "-o", "name"), + ("kubectl", "get", "--raw=/api/v1/secrets"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("helm", "get", "values", "x"), + ] + for command in commands: + assert runner.run(command, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None: + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)", + ), + {"LC_ALL": "C"}, + timeout=5, + max_bytes=1024, + ) + assert capture.returncode == 0 + assert capture.stdout_truncated and capture.stderr_truncated + assert len(capture.stdout) == len(capture.stderr) == 1024 + + +def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=0.15, + max_bytes=1024, + ) + assert capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "(os._exit(0) if p else None);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=2, + max_bytes=1024, + ) + assert not capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None: + fake = tmp_path / "kubectl" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + with pytest.raises(OSError, match="trusted paths"): + runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)}) + + +def test_attestation_checks_presence_mode_digest_and_success( + tmp_path, monkeypatch +) -> None: + fake = tmp_path / "tool" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + digest = hashlib.sha256(fake.read_bytes()).hexdigest() + monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)}) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest}) + assert ( + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256 + == digest + ) + with pytest.raises(OSError, match="unavailable"): + runner_module.attest_executable("missing", {"PATH": str(tmp_path)}) + fake.chmod(0o777) + with pytest.raises(OSError, match="unsafe ownership mode"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + fake.chmod(0o755) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64}) + with pytest.raises(OSError, match="digest"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + + +def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None: + runner, spawn, _ = build([]) + runner._attestor = lambda _command, _environment: (_ for _ in ()).throw( + OSError("no") + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:") + runner._attestor = lambda _command, _environment: runner_module.Attestation( + "/tmp/reviewer-authored/kubectl", "a" * 64 + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py index 9bb5c673..e74684d7 100644 --- a/testing/tests/test_hermes_handoff_json_rules.py +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -5,7 +5,14 @@ from __future__ import annotations import datetime as dt import json -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") @@ -23,9 +30,13 @@ def one(rule: str, expect: dict): def test_json_field_compares_paths_against_expected_values() -> None: - built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + built = one( + "json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}} + ) - assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + assert ( + evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + ) wrong = evaluate(built, {"authenticated": False, "transport": "c"}) assert wrong.status == model.FAIL @@ -40,6 +51,15 @@ def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: assert evaluate(built, "not json").status == model.NOT_RUN failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) assert failed.status == model.NOT_RUN + assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN + assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN + + +def test_json_equality_does_not_confuse_booleans_and_integers() -> None: + built = one("json_field", {"fields": {"enabled": True}}) + assert evaluate(built, {"enabled": 1}).status == model.FAIL + array = one("all_of_field", {"fields": {"enabled": False}}) + assert evaluate(array, [{"enabled": 0}]).status == model.FAIL def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: @@ -63,20 +83,32 @@ def test_json_recent_treats_stale_evidence_as_a_failure() -> None: built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) - assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + assert ( + fresh.status == model.PASS + and fresh.evidence["checked_at"]["age_seconds"] == 1800 + ) stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) assert stale.status == model.FAIL and "limit 3600s" in stale.reason naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) - assert naive.status == model.PASS + assert naive.status == model.NOT_RUN + assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN assert evaluate(built, {}).status == model.NOT_RUN + invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}}) + assert ( + evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status + == model.NOT_RUN + ) def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: - built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + built = one( + "all_of_field", + {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}, + ) payload = [ {"number": 14, "merged": True}, {"number": 15, "merged": False}, @@ -92,7 +124,10 @@ def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: - keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + keyed = one( + "all_of_field", + {"key_field": "number", "keys": [14], "fields": {"merged": True}}, + ) absent = evaluate(keyed, [{"number": 99, "merged": True}]) assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason @@ -103,7 +138,9 @@ def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: built = spec( - "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + "all_of_field", + {"step": "s", "array": "items", "fields": {"a": 1}}, + steps=(step("s"),), ) evaluation = evaluate(built, {"other": []}) @@ -124,9 +161,16 @@ def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: exempted = one( "all_of_field", - {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + { + "key_field": "name", + "exempt_keys": ["parked"], + "fields": {"suspended": False}, + }, ) - payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + payload = [ + {"name": "parked", "suspended": True}, + {"name": "live", "suspended": False}, + ] assert evaluate(exempted, payload).status == model.PASS non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) @@ -136,8 +180,13 @@ def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: def test_json_record_captures_named_paths_or_skips() -> None: - built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) - payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + built = one( + "json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}} + ) + payload = [ + {"number": 14, "head": {"ref": "a"}}, + {"number": 15, "head": {"ref": "b"}}, + ] evaluation = evaluate(built, payload) assert evaluation.status == model.PASS @@ -169,7 +218,11 @@ def routing_spec(**extra): FULL_LOG = routing_log( {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, - {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + { + "ts": "2026-08-17T11:45:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "unavailable", + }, {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, ) @@ -186,7 +239,9 @@ def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: - partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + partial = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"} + ) evaluation = evaluate(routing_spec(), partial) @@ -197,28 +252,74 @@ def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: assert "fallback:none-observed" in evaluation.reason -def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: +def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None: clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG - assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), clipped).status == model.NOT_RUN assert evaluate(routing_spec(), "").status == model.NOT_RUN assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN def test_routing_evidence_can_require_freshness() -> None: fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) - assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + assert fresh.status == model.PASS + assert max(fresh.evidence["age_seconds"].values()) <= 3600 stale = evaluate( routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG ) assert stale.status == model.FAIL and "freshness" in stale.reason - undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + undated = routing_log( + {"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"} + ) unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) assert unknown.status == model.NOT_RUN +def test_routing_freshness_is_required_per_provider_not_only_globally() -> None: + rows = routing_log( + {"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"}, + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/high", + "fallback_reason": "down", + }, + ) + evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows) + assert evaluation.status == model.FAIL + assert "freshness:provider:claude" in evaluation.reason + missing_dimension = evaluate( + routing_spec(max_age_seconds=3600, now=NOW), + routing_log( + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "down", + }, + ), + ) + assert missing_dimension.status == model.FAIL + + +@pytest.mark.parametrize( + "body", + [ + '["not-an-object"]', + '{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}', + '{"ts":"2026-08-17T11:00:00Z"}', + '{"ts":"2026-08-17T11:00:00Z","model":"bad"}', + '{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}', + ], +) +def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None: + assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN} + invalid_clock = routing_spec(now="now") + assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN + + def flux_spec(*expected: str): return spec( "flux_health", @@ -229,12 +330,14 @@ def flux_spec(*expected: str): def test_flux_health_separates_parked_objects_from_broken_ones() -> None: rows = ( - "flux-system/core\tfalse\tReady=True,Healthy=True,\n" - "flux-system/parked\ttrue\tReady=False,\n" - "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + "flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\t1\t1\ttrue\tReady=False,\n" + "flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n" ) - evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + evaluation = evaluators.evaluate( + flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)} + ) assert evaluation.status == model.FAIL assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] @@ -243,7 +346,7 @@ def test_flux_health_separates_parked_objects_from_broken_ones() -> None: def test_flux_health_flags_a_suspension_nobody_declared() -> None: - rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) @@ -251,17 +354,16 @@ def test_flux_health_flags_a_suspension_nobody_declared() -> None: assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] -def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: - """Flux drops Ready to False for the duration of a reconcile.""" +def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None: rows = ( - "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" - "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" - "flux-system/waiting\t\tReady=Unknown,\n" + "flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n" ) evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) - assert evaluation.status == model.PASS + assert evaluation.status == model.FAIL assert evaluation.evidence["total"] == 3 @@ -269,11 +371,43 @@ def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: padded = "\n\n" + FULL_LOG + "\n\n" assert evaluate(routing_spec(), padded).status == model.PASS - rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: - assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN - assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status + == model.NOT_RUN + ) + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status + == model.NOT_RUN + ) + malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n" + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status + == model.FAIL + ) + + +@pytest.mark.parametrize( + "row", + [ + "too\tfew", + "BAD NAME\t1\t1\tfalse\tReady=True,", + "flux-system/core\tx\t1\tfalse\tReady=True,", + "flux-system/core\t1\t1\tmaybe\tReady=True,", + "flux-system/core\t1\t1\tfalse\tbroken,", + "flux-system/core\t1\t1\tfalse\tReady=Yes,", + "flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,", + ], +) +def test_flux_rows_reject_malformed_columns_names_generations_and_conditions( + row: str, +) -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status + == model.NOT_RUN + ) diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py index 37c3f9b9..e3c46489 100644 --- a/testing/tests/test_hermes_handoff_model.py +++ b/testing/tests/test_hermes_handoff_model.py @@ -6,12 +6,22 @@ import datetime as dt import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) model = load_handoff_module("hermes_handoff_model") -def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): +def result( + status: str, + identifier: str = "test.check", + mandatory: bool = True, + group: str = "g", +): return model.CheckResult( spec=model.CheckSpec( id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory @@ -53,14 +63,22 @@ def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: - clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + clean = report( + result(model.PASS), errors=("the two vantages are the same principal",) + ) assert clean.decision == model.NO_GO -def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: - """An empty catalog has nothing blocking; the harness errors are what stop it.""" - assert report().decision == model.GO +def test_a_zero_state_or_unknown_status_report_is_no_go() -> None: + """Absence or an unrecognised classification can never round up to GO.""" + assert report().decision == model.NO_GO assert report(errors=("no checks were built",)).decision == model.NO_GO + unknown = result("MAYBE") + assert unknown.blocking is True + built = report(unknown) + assert built.decision == model.NO_GO + assert built.counts == dict.fromkeys(model.STATUSES, 0) + assert "groups:" in report(result(model.PASS)).render_summary() def test_blocking_results_are_ordered_failures_before_skips() -> None: @@ -93,7 +111,9 @@ def test_counts_and_group_summary_tally_every_status() -> None: def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True) + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ) ] payload = built.as_dict() @@ -138,7 +158,11 @@ def test_rationale_is_carried_only_when_a_check_has_one() -> None: def test_distinct_vantages_are_reported_in_first_use_order() -> None: built = spec( "allowed", - steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + steps=( + step("a", vantage="operator"), + step("b", vantage="self"), + step("c", vantage="operator"), + ), ) assert built.vantages == ("operator", "self") @@ -146,7 +170,9 @@ def test_distinct_vantages_are_reported_in_first_use_order() -> None: def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ), model.VantageRecord(name="self", description="d", available=False), ] diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py new file mode 100644 index 00000000..1e84cfb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -0,0 +1,51 @@ +"""Tests for the deterministic Hermes handoff mutation gate.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.quality_handoff_mutation import Mutation, MutationReport, main, run_gate + + +ROOT = Path(__file__).parents[2] + + +def test_every_security_mutant_is_killed() -> None: + report = run_gate(ROOT) + assert report.issues == () + assert report.score == 100.0 + assert report.killed == report.total >= 9 + + +def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: + path = "hermes_handoff_model.py" + cases = ( + Mutation("missing", path, "not present", "x", "pass"), + Mutation( + "baseline", + path, + "SCHEMA_VERSION = 1", + "SCHEMA_VERSION = 2", + "raise SystemExit(1)", + ), + Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"), + Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"), + ) + report = run_gate(ROOT, cases) + assert report.killed == 0 + assert len(report.issues) == 4 + assert report.score == 0.0 + + +def test_main_reports_a_perfect_score(capsys) -> None: + assert main() == 0 + assert "9/9 (100.00%)" in capsys.readouterr().out + + +def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: + monkeypatch.setattr( + "testing.quality_handoff_mutation.run_gate", + lambda _root: MutationReport(1, 0, ("survivor",)), + ) + assert main() == 1 + assert "survivor" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py index 13fdc8b4..90cef791 100644 --- a/testing/tests/test_hermes_handoff_policy.py +++ b/testing/tests/test_hermes_handoff_policy.py @@ -16,16 +16,20 @@ def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: @pytest.mark.parametrize( "argv", [ - ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"), ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - ("kubectl", "get", "secrets", "--output=name"), - ("kubectl", "logs", "deploy/x", "--tail", "1"), - ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("kubectl", "get", "nodes", "-o", "name"), + ( + "kubectl", + "get", + "pods", + "-o", + policy.projection("jsonpath={.status.phase}"), + ), ("flux", "get", "kustomizations", "--all-namespaces"), ("helm", "list", "--all-namespaces"), ("git", "merge-base", "--is-ancestor", "a", "b"), - ("git", "config", "--get", "remote.origin.url"), + ("git", "ls-remote", "origin", "refs/heads/main"), ("hermes", "kanban", "list", "--json"), ("hermes", "sessions", "list", "--source", "telegram"), ("hermes", "status"), @@ -37,40 +41,50 @@ def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: @pytest.mark.parametrize( - ("argv", "fragment"), + "argv", [ - ((), "empty command"), - (("vault", "read", "kv/x"), "forbidden binary"), - (("curl", "https://example.dev"), "forbidden binary"), - (("rm", "-rf", "/"), "allowlist"), - (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), - (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), - (("kubectl", "get", "secrets", "-A"), "-o name"), - (("kubectl", "describe", "secrets", "x"), "-o name"), - (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), - (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), - (("kubectl", "exec", "pod"), "-- command boundary"), - (("kubectl", "exec", "pod", "--"), "inner command"), - (("kubectl",), "outside the read-only boundary"), - (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), - (("flux",), "requires a subcommand"), - (("helm", "upgrade", "x"), "outside the read-only boundary"), - (("git", "commit", "-m", "x"), "not permitted"), - (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), - (("git", "config", "user.name", "x"), "may only read"), - (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), - (("hermes",), "requires a subcommand"), - ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), - ((policy.GITEA_CLIENT, "GET"), "method and a path"), - (("sh", "-c", "rm -rf /"), "frozen template"), - (("sh", "echo"), "frozen template"), - (("git", "log", "/etc/shadow"), "credential path"), + (), + ("vault", "read", "kv/x"), + ("curl", "https://example.dev"), + ("rm", "-rf", "/"), + ("kubectl", "get", "pods", "--token", "abc"), + ("kubectl", "get", "pods"), + ("kubectl", "get", "pods", "--as=system:admin"), + ("kubectl", "get", "secrets", "-o", "name"), + ("kubectl", "get", "pods,secrets/x", "-o", "name"), + ("kubectl", "describe", "pod/x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("kubectl", "get", "pods", "-o=json"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("kubectl", "port-forward", "svc/x", "80"), + ("kubectl", "exec", "pod"), + ("kubectl", "exec", "pod", "--"), + ("kubectl",), + ("flux", "reconcile", "kustomization", "x"), + ("flux",), + ("helm", "upgrade", "x"), + ("helm", "get", "values", "x"), + ("git", "commit", "-m", "x"), + ("git", "push", "origin", "HEAD:refs/heads/x"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("git", "--git-dir=/tmp/x", "status"), + ("hermes", "kanban", "complete", "t_1"), + ("hermes",), + (policy.GITEA_CLIENT, "POST", "/api/v1/x"), + (policy.GITEA_CLIENT, "GET"), + ("sh", "-c", "rm -rf /"), + ("sh", "echo"), + ("git", "log", "/etc/shadow"), ], ) -def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: - with pytest.raises(policy.PolicyError) as caught: +def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None: + with pytest.raises(policy.PolicyError): check(*argv) - assert fragment in str(caught.value) def test_unknown_mode_is_refused() -> None: @@ -78,47 +92,128 @@ def test_unknown_mode_is_refused() -> None: policy.check_argv(("kubectl", "get", "pods"), "whatever") -def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: - check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") - check("kubectl", "create", "configmap", "probe", "--dry-run=server") +def test_dry_run_mutation_commands_are_never_constructible() -> None: + with pytest.raises(policy.PolicyError, match="dry-run"): + check("kubectl", "patch", "deploy/x", "--dry-run=server") def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: """`-n get` must not make a delete look like a read.""" - with pytest.raises(policy.PolicyError, match="requires a dry run"): + with pytest.raises(policy.PolicyError): check("kubectl", "delete", "-n", "get", "configmap", "x") def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: with pytest.raises(policy.PolicyError, match="forbidden argument"): check("kubectl", "get", "namespaces", "--as", "system:admin") - check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + check( + "kubectl", + "get", + "namespaces", + "-o", + "name", + "--as", + "system:admin", + inner=True, + ) def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: - inner = policy.shell("path_readable", path="/tmp") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) - check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") - with pytest.raises(policy.PolicyError, match="allowlist"): + script = policy.render_shell("env_names") + policy._RENDERED.add(script) + check( + "kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script + ) + check( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + "--as", + "system:admin", + ) + with pytest.raises(policy.PolicyError): check("kubectl", "exec", "pod", "--", "bash", "-c", "id") -def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: - """The path scan stops at `--`; the tail is a reviewed template, not free text.""" - inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) +def test_shell_templates_reject_credential_roots() -> None: + with pytest.raises(policy.PolicyError, match="credential paths"): + policy.shell("json_fields", path="/runtime-access/token", fields="state") + with pytest.raises(policy.PolicyError, match="credential-bearing"): + policy.shell("json_fields", path="/tmp/health.json", fields="state,token") def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: - check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED) + root = "/api/v1/repos/atlas/titan-iac" + check( + policy.GITEA_CLIENT, + "POST", + f"{root}/pulls", + "--field", + "title=WIP: Hermes handoff acceptance ephemeral probe", + "--field", + f"head={ref}", + "--field", + "base=main", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + mode=policy.ARMED, + ) + check( + policy.GITEA_CLIENT, + "PATCH", + f"{root}/pulls/9", + "--field", + "state=closed", + mode=policy.ARMED, + ) + check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="may not force"): - check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + check( + "git", + "push", + "--force", + "origin", + f"HEAD:refs/heads/{ref}", + mode=policy.ARMED, + ) with pytest.raises(policy.PolicyError, match="may not force"): check("git", "push", "--all", "origin", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="not permitted"): - check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED) + + +def test_arming_and_projection_registration_fail_closed() -> None: + with pytest.raises(policy.PolicyError, match="exact ephemeral ref"): + policy.arm_ephemeral_policy("main") + policy._ARMED_REF = "" + with pytest.raises(policy.PolicyError, match="registration is malformed"): + policy.register_ephemeral_pull(9) + policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1") + + for candidate in ( + "wide", + "jsonpath={.status.token}", + "jsonpath={.metadata.annotations}", + "jsonpath={.metadata.labels.other}", + "jsonpath={.spec.containers[*].env[*].value}", + ): + with pytest.raises(policy.PolicyError): + policy.projection(candidate) + + with pytest.raises(policy.PolicyError): + check("/usr/bin/awk", "x") + with pytest.raises(policy.PolicyError): + check("git", "--exec-path=/tmp", "status") def test_every_frozen_template_renders_and_is_accepted() -> None: @@ -141,11 +236,11 @@ def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> Non with pytest.raises(policy.PolicyError, match="unknown shell template"): policy.render_shell("nope") with pytest.raises(policy.PolicyError, match="expects parameters"): - policy.render_shell("path_mode") + policy.render_shell("json_fields") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="/tmp; rm -rf /") + policy.render_shell("json_fields", path="/tmp;rm", fields="state") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="$(id)") + policy.render_shell("json_fields", path="$(id)", fields="state") def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: @@ -155,5 +250,70 @@ def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: def test_a_subcommand_must_precede_its_resource_arguments() -> None: - with pytest.raises(policy.PolicyError, match="must precede"): + with pytest.raises(policy.PolicyError): check("kubectl", "pods", "get") + + +@pytest.mark.parametrize( + "argv", + [ + ("/usr/bin/../../tmp/kubectl", "get", "pods"), + ("/tmp/reviewer/kubectl", "get", "pods"), + ("kubectl", "get", "pods", "-o", "go-template=x"), + ("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("git", "-C", "/tmp", "status"), + ("git", "--work-tree=/tmp", "status"), + (policy.GITEA_CLIENT, "GET", "not-an-api-path"), + (policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"), + ], +) +def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + with pytest.raises(policy.PolicyError, match="pinned to origin"): + check( + "git", + "push", + "upstream", + "HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"): + check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "DELETE", + "/api/v1/repos/atlas/titan-iac", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "PATCH", + "/api/v1/repos/atlas/titan-iac/pulls/8", + "--field", + "state=closed", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "POST", + "/api/v1/repos/atlas/titan-iac/pulls", + "--field", + "title=WIP: wrong branch", + "--field", + "head=main", + "--field", + "base=main", + mode=policy.ARMED, + ) diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py index a472d82b..78fe19ec 100644 --- a/testing/tests/test_hermes_handoff_redaction.py +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -63,7 +63,10 @@ def test_screened_json_is_still_parsable_json() -> None: def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: - assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + assert ( + redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") + != redaction.REDACTED + ) long_mixed = "aB3" + "x9Y2z_" * 10 assert redaction.redact(long_mixed) == redaction.REDACTED assert redaction.redact("f" * 64) == "f" * 64 @@ -108,8 +111,8 @@ def test_value_shape_gates_screening(value: str, possible: bool) -> None: def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: text, truncated = redaction.bound("x" * 100, max_bytes=10) assert truncated - assert text.startswith("x" * 10) - assert redaction.TRUNCATION_NOTE in text + assert len(text.encode()) <= 10 + assert text == redaction.TRUNCATION_NOTE[:10] text, truncated = redaction.bound("short", max_bytes=64) assert (text, truncated) == ("short", False) @@ -123,17 +126,25 @@ def test_bound_with_no_budget_keeps_nothing() -> None: def test_bound_never_splits_a_multibyte_character() -> None: text, truncated = redaction.bound("é" * 10, max_bytes=5) assert truncated - assert text.startswith("éé") + assert len(text.encode()) <= 5 -def test_safe_text_bounds_before_it_screens() -> None: - """Bounding first is what stops a huge dump from outrunning the screen.""" +def test_safe_text_screens_before_it_bounds() -> None: + """A boundary must never preserve a useful credential prefix.""" payload = "password=" + "a" * 200 text, truncated = redaction.safe_text(payload, max_bytes=20) - assert truncated + assert not truncated assert redaction.REDACTED in text +def test_no_byte_budget_exposes_a_credential_prefix() -> None: + payload = "password=A9secret-prefix-that-must-never-survive" + for budget in range(1, 64): + screened, _ = redaction.safe_text(payload, max_bytes=budget) + assert "A9" not in screened + assert len(screened.encode()) <= budget + + def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: scrubbed = redaction.scrub( { @@ -152,6 +163,25 @@ def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: assert scrubbed["note"] == ["tuple", "values"] +def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None: + scrubbed = redaction.scrub( + { + "token": 1234, + "credentials": {"x": "y"}, + "ghp_ABCDEF1234567890": "mapping-key", + "private_key": "x", + } + ) + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["credentials"] == redaction.REDACTED + assert scrubbed["private_key"] == redaction.REDACTED + assert not any("ghp_" in key for key in scrubbed) + + +def test_unterminated_private_key_blocks_are_screened() -> None: + assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc") + + def test_screening_is_idempotent() -> None: once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") assert redaction.redact(once) == once diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py index 11f09840..03d5785f 100644 --- a/testing/tests/test_hermes_handoff_rules.py +++ b/testing/tests/test_hermes_handoff_rules.py @@ -4,7 +4,12 @@ from __future__ import annotations import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) rules = load_handoff_module("hermes_handoff_rules") model = load_handoff_module("hermes_handoff_model") @@ -33,15 +38,18 @@ def test_dotted_paths_index_and_map(path: str, expected) -> None: assert rules.dotted(PAYLOAD, path) == expected -@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +@pytest.mark.parametrize( + "path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"] +) def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: with pytest.raises(KeyError): rules.dotted(PAYLOAD, path) -def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: - """Kanban bodies routinely carry raw newlines; that is not a broken read.""" - assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" +def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None: + """Only RFC-compliant, nonempty JSON is accepted as release evidence.""" + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout='{"title": "a\nb"}')) with pytest.raises(ValueError, match="empty output"): rules.parse_json(outcome(stdout=" ")) with pytest.raises(ValueError): @@ -59,10 +67,17 @@ def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: assert never_run.status == model.NOT_RUN assert "never executed" in never_run.reason - errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + errored = rules.missing_steps( + built, {"a": outcome(error="timeout"), "b": outcome()} + ) assert errored.status == model.NOT_RUN assert "did not run" in errored.reason + truncated = rules.missing_steps( + built, {"a": outcome(truncated=True), "b": outcome()} + ) + assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None @@ -81,7 +96,9 @@ def test_an_optional_step_may_be_absent() -> None: ("", 0, False), ], ) -def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: +def test_denial_detection_needs_a_refusal_not_merely_a_failure( + stderr, returncode, denied +) -> None: assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied @@ -105,7 +122,10 @@ def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: assert failed.status == model.NOT_RUN and "failed" in failed.reason unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) - assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + assert ( + unparsable.status == model.NOT_RUN + and "did not return JSON" in unparsable.reason + ) def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: @@ -128,6 +148,17 @@ def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: del rules.EVALUATORS["explodes-for-test"] +def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None: + @rules.evaluator("unknown-status-for-test") + def _unknown(_spec, _outcomes): + return rules.Evaluation("MAYBE", "bad") + + evaluation = rules.evaluate(spec("unknown-status-for-test"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown status" in evaluation.reason + del rules.EVALUATORS["unknown-status-for-test"] + + def test_is_list_distinguishes_arrays_from_strings() -> None: assert rules.is_list([1]) and rules.is_list(()) assert not rules.is_list("text") diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py index 379bb27d..881ffd55 100644 --- a/testing/tests/test_hermes_handoff_run.py +++ b/testing/tests/test_hermes_handoff_run.py @@ -4,6 +4,7 @@ from __future__ import annotations import datetime as dt import subprocess +from dataclasses import replace from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -22,20 +23,35 @@ TARGETS = catalog.Targets(now=NOW) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def whoami(username: str) -> object: - return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + return completed(stdout=username) def runner_for(results: list, **kwargs): return exec_module.Runner( - clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + clock=FakeClock(), + spawn=FakeSpawn(results), + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, ) -def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): +def check( + rule: str, + expect: dict | None = None, + steps=(), + identifier="test.check", + mandatory=True, +): return model.CheckSpec( id=identifier, title="t", @@ -51,8 +67,23 @@ def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: specs = harness_run.build_catalog(TARGETS) assert harness_run.validate_catalog(specs) == [] groups = {spec.group for spec in specs} - assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups - assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + assert { + "baseline", + "identity", + "routing", + "scopes", + "access-denied", + "gitops", + } <= groups + assert { + "forge", + "nodes", + "build", + "reliability", + "pool", + "surfaces", + "ephemeral", + } <= groups def test_catalog_validation_names_every_structural_problem() -> None: @@ -61,14 +92,20 @@ def test_catalog_validation_names_every_structural_problem() -> None: check("allowed", identifier="a.dup"), check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), - check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + check( + "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only" + ), model.CheckSpec( id="e.impersonation", title="t", group="g", rule="allowed", steps=( - model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "get", "ns", "--as", "x"), + ), ), ), ] @@ -83,23 +120,81 @@ def test_catalog_validation_names_every_structural_problem() -> None: assert "impersonates from the operator vantage" in problems +def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None: + broken = replace( + TARGETS, + repo="atlas/other", + remote="upstream", + reviewed_pr_number=20, + reviewed_head_ref="other", + baseline_commit="bad", + remote_main_sha="bad", + reviewed_head_sha="bad", + build_sha="bad", + agent_image="latest", + deployment_revision="0", + node_count=1, + pool_replicas=1, + expect_telegram_sessions=False, + chat_config_revision="", + pool_worker_env=(), + dependency_heads=((14, "bad"), (14, "bad")), + max_evidence_age_seconds=0, + ) + problems = harness_run.validate_targets(broken) + assert len(problems) >= 12 + sha = "1" * 40 + valid = replace( + TARGETS, + remote_main_sha=sha, + reviewed_head_sha=sha, + build_sha=sha, + agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}", + deployment_revision="1", + chat_config_revision="rev", + dependency_heads=tuple( + (number, sha) for number in TARGETS.dependency_pull_requests + ), + ) + assert harness_run.validate_targets(valid) == [] + wrong_image_source = replace( + valid, + agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}", + ) + assert "image tag/digest" in " ".join( + harness_run.validate_targets(wrong_image_source) + ) + wrong_build = replace(valid, build_sha="2" * 40) + assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build)) + malformed_heads = replace( + valid, + dependency_heads=tuple( + (number, "bad") for number in valid.dependency_pull_requests + ), + ) + assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads)) + + def test_identity_resolution_extracts_only_the_username() -> None: runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) - username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + username, detail = harness_run.resolve_identity( + runner, exec_module.operator_vantage() + ) assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") failed = harness_run.resolve_identity( - runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + runner_for([completed(stderr="no access", returncode=1)]), + exec_module.operator_vantage(), ) assert failed[0] == "" and "no access" in failed[1] unparsable = harness_run.resolve_identity( - runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + runner_for([completed(stdout="two names")]), exec_module.operator_vantage() ) - assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + assert unparsable[0] == "" and "no username" in unparsable[1] nameless = harness_run.resolve_identity( - runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + runner_for([completed(stdout="")]), exec_module.operator_vantage() ) assert nameless[0] == "" and "no username" in nameless[1] @@ -119,7 +214,12 @@ def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: vantages, records = harness_run.resolve_vantages(runner, TARGETS) by_name = {record.name: record for record in records} - assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert set(vantages) == { + catalog.OPERATOR, + catalog.SELF, + catalog.SWITCHYARD, + catalog.CHAT, + } assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick assert by_name[catalog.NODE].available is False @@ -133,7 +233,9 @@ def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: assert set(vantages) == {catalog.OPERATOR} assert records[0].available is False - assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + assert harness_run.vantage_problems(records)[0].startswith( + "the operator vantage is unavailable" + ) def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: @@ -176,8 +278,14 @@ def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> N def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: """Corroboration needs two identities; one identity twice is a restatement.""" records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="same", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="same", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] problems = harness_run.vantage_problems(records) @@ -195,7 +303,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: assert "in-pod Hermes vantage is unavailable" in absent[0] anonymous = harness_run.vantage_problems( - [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + [ + operator, + model.VantageRecord(name=catalog.SELF, description="d", available=True), + ] ) assert "did not report an identity" in anonymous[0] @@ -205,6 +316,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: model.VantageRecord( name=catalog.SELF, description="d", identity="agent", available=True ), + model.VantageRecord( + name=catalog.SWITCHYARD, description="d", available=True + ), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] ) assert clean == [] @@ -224,11 +339,20 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None spec = check( "stdout_matches", {"step": "s", "equals": "locked"}, - steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "version"), + record=False, + ), + ), ) runner = runner_for([completed(stdout="locked", stderr="noise")]) - result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + result = harness_run.run_check( + runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()} + ) assert result.status == model.PASS assert result.outcomes[0].stdout == harness_run.UNRECORDED @@ -237,7 +361,15 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None def test_an_expired_deadline_leaves_the_report_saying_so() -> None: clock = FakeClock() - runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + runner = exec_module.Runner( + clock=clock, + spawn=FakeSpawn([]), + environ={}, + deadline_seconds=1.0, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) clock.advance(5.0) spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) @@ -259,12 +391,24 @@ def test_an_expired_deadline_leaves_the_report_saying_so() -> None: def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) specs = [ - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + check( + "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last" + ), + check( + "allowed", + steps=(step("s", vantage=catalog.OPERATOR),), + identifier="a.first", + ), ] records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] report = harness_run.build_report( @@ -279,16 +423,23 @@ def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> No assert [result.spec.id for result in report.results] == ["a.first", "z.last"] assert report.baseline["baseline_commit"] == TARGETS.baseline_commit - assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.baseline["dependency_pull_requests"] == list( + TARGETS.dependency_pull_requests + ) assert report.decision == model.GO assert report.finished_at def test_armed_results_replace_the_matching_catalog_placeholders() -> None: - placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + placeholder = check( + "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False + ) armed = model.CheckResult( spec=model.CheckSpec( - id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + id="ephemeral.feature-branch-push", + title="t", + group="ephemeral", + rule="armed", ), status=model.PASS, reason="pushed", diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py index 64964a27..f62ead15 100644 --- a/testing/tests/test_hermes_handoff_support.py +++ b/testing/tests/test_hermes_handoff_support.py @@ -33,6 +33,7 @@ def outcome( stderr: str = "", returncode: int | None = 0, error: str | None = None, + truncated: bool = False, vantage: str = "self", argv: tuple[str, ...] = ("kubectl", "get", "pods"), ) -> Any: @@ -44,6 +45,7 @@ def outcome( stdout=stdout, stderr=stderr, error=error, + truncated=truncated, ) @@ -68,12 +70,14 @@ def spec( ) -def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: +def step( + key: str, kind: str | None = None, vantage: str = "self", optional: bool = False +) -> Any: """Return a synthetic step.""" return model.Step( key=key, vantage=vantage, - argv=("kubectl", "get", "pods"), + argv=("kubectl", "get", "pods", "-o", "name"), kind=kind or model.READ, optional=optional, ) diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..fecae836 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -2,13 +2,17 @@ from __future__ import annotations +import hashlib from pathlib import Path import textwrap from testing.quality_contract import load_contract from testing.quality_coverage import run_check as run_coverage_check from testing.quality_docs import run_check as run_docs_check -from testing.quality_hygiene import run_check as run_hygiene_check +from testing.quality_hygiene import ( + count_files_over_line_limit, + run_check as run_hygiene_check, +) def test_bundled_contract_exposes_local_and_jenkins_profiles(): @@ -18,6 +22,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles(): assert contract["pytest_suites"]["unit"]["paths"] +def test_handoff_modules_are_exactly_managed_linted_and_covered(): + """The canonical contract must not silently drop a handoff module.""" + contract = load_contract() + expected = { + path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") + } + assert len(expected) == 15 + assert expected <= set(contract["managed_modules"]) + assert expected <= set(contract["lint_paths"]) + assert expected <= set(contract["coverage"]["tracked_files"]) + assert expected <= set(contract["coverage"]["branch_tracked_files"]) + assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"] + assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"] + + def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path): module_path = tmp_path / "managed.py" module_path.write_text("value = 1\n", encoding="utf-8") @@ -43,7 +62,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): (tmp_path / "README.md").write_text("", encoding="utf-8") contract = { - "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}], + "required_docs": [ + {"path": "README.md", "description": ""}, + {"path": "missing.md", "description": "Missing"}, + ], "managed_modules": [], "lint_paths": [], "pytest_suites": {"unit": {"description": "", "paths": []}}, @@ -62,6 +84,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): tests_dir = tmp_path / "tests" tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8") bad_name = tests_dir / "bad-name.py" bad_name.write_text("x = 1\n", encoding="utf-8") long_file = tests_dir / "test_too_long.py" @@ -84,7 +107,34 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): issues = run_hygiene_check(contract, tmp_path) assert any("file exceeds 3 LOC" in issue for issue in issues) - assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues) + assert any( + "naming rule failed" in issue and "bad-name.py" in issue for issue in issues + ) + + +def test_hygiene_legacy_exception_is_exact_and_invalidated_by_any_edit(tmp_path: Path): + legacy = tmp_path / "legacy.py" + legacy.write_text("line\n" * 4, encoding="utf-8") + contract = { + "hygiene": { + "max_lines": 3, + "line_limit_globs": ["*.py"], + "legacy_line_exceptions": { + "legacy.py": { + "lines": 4, + "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), + } + }, + } + } + assert run_hygiene_check(contract, tmp_path) == [] + assert count_files_over_line_limit(contract, tmp_path) == 0 + + legacy.write_text("edit\n" * 4, encoding="utf-8") + assert run_hygiene_check(contract, tmp_path) == [ + "file exceeds 3 LOC: legacy.py (4)" + ] + assert count_files_over_line_limit(contract, tmp_path) == 1 def test_coverage_check_enforces_per_file_floor(tmp_path: Path): @@ -124,9 +174,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path): missing_xml = tmp_path / "missing.xml" - assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [ - "coverage xml missing: missing.xml" - ] + assert run_coverage_check( + {"coverage": {"tracked_files": []}}, tmp_path, missing_xml + ) == ["coverage xml missing: missing.xml"] source_dir = tmp_path / "pkg" source_dir.mkdir() @@ -143,7 +193,7 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..a42df4ed 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,59 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_explicit_branch_floor(tmp_path: Path) -> None: + """Branch-tracked modules must report and meet the configured floor.""" + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["low.py", "missing.py", "legacy.py"], + "branch_tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + passing = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["passing.py"], + "branch_tracked_files": ["passing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert passing == [] + + +def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples( + tmp_path: Path, +) -> None: + xml = tmp_path / "coverage.xml" + xml.write_text("", encoding="utf-8") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml + ) + == 0.0 + ) diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..ef59abd0 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_sonar_issue_never_ends_before_its_start_line() -> None: + issue = semgrep_report._sonar_issue_from_finding( + {"path": "app.py", "start": {"line": 10}, "end": {"line": 2}} + ) + assert issue is not None + assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10} + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" @@ -108,7 +116,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: list_json = tmp_path / "list.json" list_json.write_text("[]", encoding="utf-8") - assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith( + "invalid JSON:" + ) assert semgrep_report._read_json(list_json) == { "errors": [{"message": "report payload is not an object"}], "results": [], @@ -137,16 +147,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["errors_total"] == 1 - sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + sonar_payload = json.loads( + (tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8") + ) assert sonar_payload == {"issues": []} -def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: +def test_main_honors_custom_blocking_severity_without_sonar_output( + tmp_path: Path, +) -> None: """CLI callers can choose stricter blocking severities without writing Sonar issues.""" semgrep_json = tmp_path / "semgrep.json" semgrep_json.write_text( - json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + json.dumps( + { + "results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], + "errors": [], + } + ), encoding="utf-8", ) output = tmp_path / "semgrep-report.json" From c808baff40a12a90f43c2e5e39b94d139f88ea29 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 13:12:36 +0000 Subject: [PATCH 15/46] docs(hermes): record repaired conflict graph --- docs/hermes_full_handoff_acceptance.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index d7aa9608..be13c1f0 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -195,6 +195,11 @@ each step: #12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, and all desired-state changes during conflict resolution. +An exact post-repair merge-tree check against the heads in the table reports +#19 clean with current `main`, #12, #17, and #20, and conflicting with #14, +#15, #16, and #18. Recompute this graph from the Gitea API immediately before +integration; do not infer conflict status from Gitea's draft `mergeable` field. + The #16/#19 integration must keep both the immutable digest and the source tag (`newTag: git--build-` plus the digest) in rendered image identity. Digest-only rendering loses the attested build-source link and therefore From b080b5f622ae0998213f3287762aea30dc931a73 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 14:49:33 +0000 Subject: [PATCH 16/46] hermes: finalize completed review verdicts in the goal judge The local goal judge scored every worker report against "did the reviewed implementation reach a shippable state". A read-only reviewer that returned a completed BLOCK verdict with findings was therefore resumed turn after turn with an instruction to repair code it was forbidden to touch (observed live on t_dbdcd739), burning subscription capacity and risking an unbounded loop. Completion is now judged against the action the card assigned: * Cards declare their role explicitly with Hermes-Task-Role / Hermes-Expected- Output metadata. Pre-contract cards fall back to a narrow inference that needs a read-only scope, a requested verdict, no requested mutation deliverable, and a report that changed no files. * Role resolution reads only the card itself. Prior attempts, parent results, cross-task history and comments appended to the worker context can no longer reassign the role. * Review, audit and diagnostic cards finalize deterministically on a truthful SHIP or BLOCK verdict with evidence, and fail closed on a missing, unrecognized or self-contradictory verdict, on a BLOCK without findings, and on a verdict without evidence. Every rejection reason carries the read-only guard, so a resumed review is never told to edit the reviewed implementation. * Implementation cards keep the fail-closed model judge unchanged, including the unfinished-work heuristic and judge-unavailable rejection. * All judge reasons are bounded, single-line and secret-redacted before they reach Kanban metadata, comments and continuation prompts. Co-Authored-By: Claude Opus 5 --- services/hermes/scripts/cli_lane_goal.py | 288 +++++++++++- testing/tests/test_hermes_cli_review_goal.py | 461 +++++++++++++++++++ testing/tests/test_hermes_cli_review_lane.py | 427 +++++++++++++++++ 3 files changed, 1157 insertions(+), 19 deletions(-) create mode 100644 testing/tests/test_hermes_cli_review_goal.py create mode 100644 testing/tests/test_hermes_cli_review_lane.py diff --git a/services/hermes/scripts/cli_lane_goal.py b/services/hermes/scripts/cli_lane_goal.py index cc96ea79..0cf991f5 100644 --- a/services/hermes/scripts/cli_lane_goal.py +++ b/services/hermes/scripts/cli_lane_goal.py @@ -1,5 +1,24 @@ #!/usr/bin/env python3 -"""Fail-closed completion checks for durable external Kanban workers.""" +"""Role-aware completion checks for durable external Kanban workers. + +Completion is judged against the action the card assigned, not against the +health of whatever the card asked the worker to look at. + +* Implementation cards keep the fail-closed local goal judge: a report is + accepted only when the model agrees every explicit requirement finished. +* Review, audit, and diagnostic cards deliver a *verdict*. A completed report + carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card + even when the verdict says the reviewed artifact must not ship. Findings are + the deliverable, not a task blocker, so the card is never resumed with an + instruction to repair an implementation the reviewer may not touch. + +The task role comes from explicit card metadata (``Hermes-Task-Role: review``) +whenever the card author supplies it. Cards written before that contract fall +back to a deliberately narrow inference that requires a read-only scope, a +requested verdict, no requested mutation deliverable, and a report that changed +no files. Review completion is fail-closed on a missing, unrecognized, or +self-contradictory verdict and on missing evidence. +""" from __future__ import annotations @@ -24,6 +43,30 @@ GOAL_JUDGE_TIMEOUT_SECONDS = max( min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))), ) RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"}) +REVIEW_ROLE = "review" +IMPLEMENTATION_ROLE = "implementation" +TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE}) +ROLE_ALIASES = { + "audit": REVIEW_ROLE, + "assessment": REVIEW_ROLE, + "diagnosis": REVIEW_ROLE, + "diagnostic": REVIEW_ROLE, + "inspection": REVIEW_ROLE, + "review": REVIEW_ROLE, + "reviewer": REVIEW_ROLE, + "verdict": REVIEW_ROLE, + "build": IMPLEMENTATION_ROLE, + "change": IMPLEMENTATION_ROLE, + "implement": IMPLEMENTATION_ROLE, + "implementation": IMPLEMENTATION_ROLE, + "repair": IMPLEMENTATION_ROLE, +} +JUDGE_REASON_LIMIT = 600 +MIN_REVIEW_SUMMARY_CHARS = 80 +READ_ONLY_GUARD = ( + "the assigned action is a read-only review: report the verdict and evidence, " + "and do not modify the reviewed implementation" +) GOAL_JUDGE_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, @@ -43,6 +86,60 @@ UNFINISHED_EVIDENCE = re.compile( r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)", re.IGNORECASE | re.DOTALL, ) +# The worker context appends prior attempts, parent results, cross-task history +# and comments after the card the worker was actually given. Role resolution +# reads only the card itself so an unrelated sentence quoted from an earlier run +# cannot reassign the task role. +HISTORY_SECTION = re.compile( + r"^##\s+(?:Prior attempts\b|Parent task results\b|Recent work by\b|Comment thread\b)", + re.IGNORECASE | re.MULTILINE, +) +# Explicit, machine-readable card metadata. This is the supported contract; +# every pattern below it only keeps pre-contract cards working. +ROLE_DIRECTIVE = re.compile( + r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$", + re.IGNORECASE | re.MULTILINE, +) +OUTPUT_DIRECTIVE = re.compile( + r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$", + re.IGNORECASE | re.MULTILINE, +) +READ_ONLY_SCOPE = re.compile( + r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?" + r"(?:edit|modify|change|alter|patch|rewrite|implement|repair)\b" + r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b" + r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b" + r"|\bno\s+(?:code|file|source|implementation)\s+(?:edits?|changes?|modifications?)\b" + r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b" + r"|\bread[\s-]?only\s+(?:scope|review|audit|analysis|assessment|inspection|diagnos\w+)\b)", + re.IGNORECASE, +) +VERDICT_DELIVERABLE = re.compile( + r"(?:\bship\s+or\s+block\b|\bblock\s+or\s+ship\b|\bship\s*/\s*block\b" + r"|\breturn\s+(?:a\s+|one\s+|the\s+)?(?:strict\s+)?verdict\b" + r"|\b(?:review|audit|release|diagnostic)\s+verdict\b|\bverdict\s*[:=])", + re.IGNORECASE, +) +MUTATION_DELIVERABLE = re.compile( + r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b" + r"|\bpush\s+(?:the\s+|your\s+|one\s+)?(?:branch|commit|change\w*|fix\w*)\b" + r"|\bcommit\s+and\s+push\b" + r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)" + r"|\bdraft\s+pr\s+only\b)", + re.IGNORECASE, +) +VERDICT_DIRECTIVE = re.compile( + r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*" + r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b", + re.IGNORECASE, +) +VERDICT_TOKEN = re.compile(r"(? str: @@ -55,17 +152,138 @@ def _bounded(value: str, limit: int) -> str: return f"{value[:head]}{marker}{value[-(remaining - head):]}" -def unfinished_result_reason(result: dict[str, Any]) -> str | None: +def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str: + """Return one bounded, single-line, secret-free judge reason.""" + text = SECRET_LIKE.sub("[redacted]", str(value or "")) + text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split()) + if not text: + return "local judge supplied no reason" + return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." + + +def _strings(value: Any) -> list[str]: + """Return the non-empty string entries of a reported evidence list.""" + if not isinstance(value, list): + return [] + return [str(item).strip() for item in value if str(item).strip()] + + +def card_scope(objective: str) -> str: + """Return the card the worker was assigned, without appended run history.""" + text = str(objective or "") + match = HISTORY_SECTION.search(text) + return text[: match.start()] if match else text + + +def _declared_role(objective: str) -> str | None: + """Return the role a card declared through explicit worker metadata.""" + for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE): + roles = { + ROLE_ALIASES[value] + for value in ( + match.group(1).strip().lower() for match in pattern.finditer(objective) + ) + if value in ROLE_ALIASES + } + if len(roles) == 1: + return roles.pop() + if roles: + # A card that declares two different roles has no usable contract. + # Hold it to the stricter implementation regime rather than letting + # the more permissive review path win an ambiguous declaration. + return IMPLEMENTATION_ROLE + return None + + +def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]: + """Resolve the assigned task role and how that resolution was reached.""" + text = card_scope(objective) + declared = _declared_role(text) + if declared is not None: + return declared, "directive" + changed = _strings((result or {}).get("changed_files")) + if ( + not changed + and READ_ONLY_SCOPE.search(text) + and VERDICT_DELIVERABLE.search(text) + and not MUTATION_DELIVERABLE.search(text) + ): + return REVIEW_ROLE, "inferred" + return IMPLEMENTATION_ROLE, "default" + + +def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]: + """Return the declared SHIP/BLOCK verdict, or why it cannot be trusted.""" + declared: set[str] = set() + field = result.get("verdict") + if field is not None: + value = str(field).strip().upper() + if value not in {"SHIP", "BLOCK"}: + return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}" + declared.add(value) + summary = str(result.get("summary") or "") + declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary)) + if len(declared) > 1: + return None, "review report declares both SHIP and BLOCK verdicts" + if declared: + return declared.pop(), None + tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)} + if len(tokens) > 1: + return None, "review summary names both SHIP and BLOCK without a verdict line" + if tokens: + return tokens.pop(), None + return None, "review report declares no explicit SHIP or BLOCK verdict" + + +def review_completion_problem(result: dict[str, Any]) -> str | None: + """Fail closed on a review report that does not carry a usable verdict.""" + verdict, problem = review_verdict(result) + if problem or verdict is None: + return problem or "review report declares no explicit SHIP or BLOCK verdict" + summary = str(result.get("summary") or "").strip() + if len(summary) < MIN_REVIEW_SUMMARY_CHARS: + return f"{verdict} verdict carries no reviewable rationale in its summary" + findings = _strings(result.get("findings")) + if verdict == "BLOCK" and not findings: + return "BLOCK verdict reports no findings to justify it" + if not findings and not _strings(result.get("tests_run")) and not _strings( + result.get("artifacts") + ): + return f"{verdict} verdict reports no findings, tests, or artifacts as evidence" + return None + + +def _reported_role(result: dict[str, Any]) -> str: + """Classify a report shape when no objective is available to the caller.""" + if _strings(result.get("changed_files")): + return IMPLEMENTATION_ROLE + if result.get("verdict") is not None or VERDICT_DIRECTIVE.search( + str(result.get("summary") or "") + ): + return REVIEW_ROLE + return IMPLEMENTATION_ROLE + + +def unfinished_result_reason( + result: dict[str, Any], + *, + role: str | None = None, +) -> str | None: """Reject self-contradictory completion reports without model inference.""" status = str(result.get("status") or "") summary = str(result.get("summary") or "").strip() blockers = result.get("blockers") if status == "incomplete": - return summary or "worker explicitly reported incomplete work" + return sanitize_reason(summary or "worker explicitly reported incomplete work") if status != "completed": return None if isinstance(blockers, list) and any(str(item).strip() for item in blockers): return "worker reported blockers while claiming completion" + if (role or _reported_role(result)) == REVIEW_ROLE: + # A review's prose describes the reviewed artifact, so the unfinished + # work heuristic below would read the artifact's state as the review's + # own. The verdict contract is the deterministic gate instead. + return review_completion_problem(result) tests = result.get("tests_run") evidence = [summary] if isinstance(tests, list): @@ -76,18 +294,14 @@ def unfinished_result_reason(result: dict[str, Any]) -> str | None: return None -def judge_goal_completion( - objective: str, - result: dict[str, Any], - *, - url: str = GOAL_JUDGE_URL, - open_request: Callable[..., Any] = urllib.request.urlopen, -) -> tuple[bool, str]: - """Use the local model as a fail-closed judge for explicit goal cards.""" - deterministic_reason = unfinished_result_reason(result) - if deterministic_reason: - return False, deterministic_reason - payload = { +def _completion_claimed(result: dict[str, Any]) -> bool: + """Report whether the worker actually claimed a finished task.""" + return str(result.get("status") or "") == "completed" + + +def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]: + """Build the fail-closed local judge request for an implementation card.""" + return { "model": GOAL_JUDGE_MODEL, "messages": [ { @@ -98,13 +312,16 @@ def judge_goal_completion( "Return complete only when the report gives concrete evidence that every " "explicit requirement finished. Return continue when any work, test, command, " "commit, push, review, or verification is pending, in progress, omitted, or " - "inconclusive. Do not trust the report status field." + "inconclusive. Defects the worker reports about some other artifact are " + "findings, not unfinished work on this task. Do not trust the report status " + "field." ), }, { "role": "user", "content": json.dumps( { + "task_role": role, "objective": _bounded(objective, 6000), "worker_report": _bounded( json.dumps(result, default=str), 4000 @@ -126,9 +343,40 @@ def judge_goal_completion( }, }, } + + +def judge_goal_completion( + objective: str, + result: dict[str, Any], + *, + url: str = GOAL_JUDGE_URL, + open_request: Callable[..., Any] = urllib.request.urlopen, +) -> tuple[bool, str]: + """Judge one worker report against the action its card actually assigned.""" + role, source = task_role(objective, result) + deterministic_reason = unfinished_result_reason(result, role=role) + if deterministic_reason is None and not _completion_claimed(result): + deterministic_reason = ( + f"worker reported status {str(result.get('status') or 'unknown')!r} " + "rather than a finished task" + ) + if deterministic_reason: + if role == REVIEW_ROLE: + return False, sanitize_reason(f"{READ_ONLY_GUARD}; {deterministic_reason}") + return False, sanitize_reason(deterministic_reason) + if role == REVIEW_ROLE: + # The requested deliverable is present and self-consistent. A verdict + # that the reviewed artifact must not ship is a finished review, so the + # card finalizes instead of resuming a worker that may not repair it. + verdict, _ = review_verdict(result) + findings = len(_strings(result.get("findings"))) + return True, sanitize_reason( + f"review deliverable complete: {verdict} verdict with {findings} finding(s); " + f"task role {role} resolved by {source}" + ) request = urllib.request.Request( url, - data=json.dumps(payload).encode("utf-8"), + data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) @@ -139,7 +387,7 @@ def judge_goal_completion( verdict = json.loads(content) if verdict.get("verdict") not in {"complete", "continue"}: raise ValueError("local judge returned an invalid verdict") - reason = str(verdict.get("reason") or "local judge supplied no reason") + reason = sanitize_reason(verdict.get("reason")) return verdict.get("verdict") == "complete", reason except ( AttributeError, @@ -151,4 +399,6 @@ def judge_goal_completion( json.JSONDecodeError, urllib.error.URLError, ) as error: - return False, f"local completion judge unavailable: {type(error).__name__}: {error}" + return False, sanitize_reason( + f"local completion judge unavailable: {type(error).__name__}: {error}" + ) diff --git a/testing/tests/test_hermes_cli_review_goal.py b/testing/tests/test_hermes_cli_review_goal.py new file mode 100644 index 00000000..05656330 --- /dev/null +++ b/testing/tests/test_hermes_cli_review_goal.py @@ -0,0 +1,461 @@ +"""Task-role and verdict-contract tests for the Hermes goal judge.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/cli_lane_goal.py" +SPEC = importlib.util.spec_from_file_location("cli_lane_review_goal_test", SCRIPT) +assert SPEC and SPEC.loader +goal = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = goal +SPEC.loader.exec_module(goal) + + +REVIEW_CARD = """# Kanban task t_review: Independent read-only review of the worker pool + +Assignee: cli-claude-xhigh + +## Body +Perform a fresh independent read-only release review of the pool implementation. +Do not edit files, comment, commit, push, merge, publish, or deploy. +Return strict SHIP or BLOCK with exact file/line and reproducible inputs. +""" +IMPLEMENTATION_CARD = """# Kanban task t_impl: Repair the pool blockers + +Assignee: cli-auto + +## Body +Repair every reported blocker, run the focused suites, and open a draft PR. +Return exact pushed SHA and evidence. +""" +POLLUTED_CARD = REVIEW_CARD + """ +## Recent work by @cli-claude-xhigh +- t_prior: Implemented and pushed the pool. PR #18 is open at the exact pushed SHA. + +## Comment thread +- shepherd: commit and push the repairs before returning. +""" + + +def _review(**overrides): + value = { + "status": "completed", + "summary": ( + "Independent read-only review of PR #18 at head 2000252. Verdict: BLOCK. " + "Five reproducible P0 defects fire on the coordinator's first tick and the " + "review worktree was left pristine." + ), + "changed_files": [], + "tests_run": ["pytest testing/tests/test_pool.py: 82 passed"], + "artifacts": ["/opt/data/workspace/evidence/t_review/RUN.md"], + "findings": ["P0-1 BLOCKER - coordinator.py:248 compares int to str run ids."], + "blockers": [], + } + value.update(overrides) + return value + + +def _implementation(**overrides): + value = { + "status": "completed", + "summary": "All acceptance criteria passed and the branch was pushed.", + "changed_files": ["src/example.py"], + "tests_run": ["pytest -q: 12 passed"], + "artifacts": [], + "findings": [], + "blockers": [], + } + value.update(overrides) + return value + + +def _no_judge(*_args, **_kwargs): + raise AssertionError("the model judge must not be consulted for a review card") + + +def test_completed_block_review_finalizes_without_consulting_the_model(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review(), + open_request=_no_judge, + ) + + assert accepted is True + assert "BLOCK verdict with 1 finding(s)" in reason + assert "resolved by inferred" in reason + + +def test_completed_ship_review_finalizes_on_its_own_evidence(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review( + summary=( + "Independent read-only review of PR #18 at head 2000252. Verdict: SHIP. " + "Every prioritized boundary was exercised and no defect survived." + ), + findings=[], + ), + open_request=_no_judge, + ) + + assert accepted is True + assert "SHIP verdict with 0 finding(s)" in reason + + +@pytest.mark.parametrize( + ("role_line", "expected"), + [ + ("Hermes-Task-Role: review", goal.REVIEW_ROLE), + ("Task-Role: audit", goal.REVIEW_ROLE), + ("- Hermes Task Role = diagnostic", goal.REVIEW_ROLE), + ("Hermes-Expected-Output: verdict", goal.REVIEW_ROLE), + ("Hermes-Task-Role: implementation", goal.IMPLEMENTATION_ROLE), + ], +) +def test_explicit_card_metadata_decides_the_task_role(role_line, expected): + card = IMPLEMENTATION_CARD.replace("## Body\n", f"## Body\n{role_line}\n") + + assert goal.task_role(card, _review())[0] == expected + assert goal.task_role(card, _review())[1] == "directive" + + +def test_explicit_review_metadata_survives_a_changed_file_report(): + card = REVIEW_CARD.replace("## Body\n", "## Body\nHermes-Task-Role: review\n") + + accepted, reason = goal.judge_goal_completion( + card, + _review(changed_files=["reports/review.md"]), + open_request=_no_judge, + ) + + assert accepted is True + assert "resolved by directive" in reason + + +def test_appended_run_history_cannot_reassign_the_card_role(): + assert goal.task_role(POLLUTED_CARD, _review()) == (goal.REVIEW_ROLE, "inferred") + assert "Recent work by" not in goal.card_scope(POLLUTED_CARD) + assert "Do not edit files" in goal.card_scope(POLLUTED_CARD) + + +def test_implementation_card_never_infers_a_review_role(): + assert goal.task_role(IMPLEMENTATION_CARD, _review()) == ( + goal.IMPLEMENTATION_ROLE, + "default", + ) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + (_review(summary="Verdict: BLOCK."), "carries no reviewable rationale"), + (_review(findings=[]), "BLOCK verdict reports no findings"), + ( + _review(summary=_review()["summary"].replace("Verdict: BLOCK.", "")), + "declares no explicit SHIP or BLOCK verdict", + ), + ( + _review(verdict="SHIP"), + "declares both SHIP and BLOCK verdicts", + ), + ( + _review(verdict="maybe"), + "review verdict field is not SHIP or BLOCK", + ), + ( + _review( + summary=( + "The reviewed change is fine overall. SHIP is defensible but BLOCK " + "is also arguable given the unresolved capacity question." + ), + ), + "names both SHIP and BLOCK without a verdict line", + ), + ( + _review( + summary=( + "Independent read-only review of PR #18. Verdict: SHIP. Every " + "prioritized boundary was exercised and nothing survived." + ), + findings=[], + tests_run=[], + artifacts=[], + ), + "no findings, tests, or artifacts as evidence", + ), + ], +) +def test_malformed_review_completion_fails_closed(result, expected): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + result, + open_request=_no_judge, + ) + + assert accepted is False + assert expected in reason + assert reason.startswith(goal.READ_ONLY_GUARD) + + +def test_rejected_review_never_demands_an_implementation_edit(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review(findings=[]), + open_request=_no_judge, + ) + + assert accepted is False + assert "do not modify the reviewed implementation" in reason + + +def test_incomplete_review_turn_is_resumed_under_the_read_only_guard(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review(status="incomplete", summary="Only two of six boundaries were read."), + open_request=_no_judge, + ) + + assert accepted is False + assert reason.startswith(goal.READ_ONLY_GUARD) + assert "Only two of six boundaries were read." in reason + + +def test_blocked_review_reports_its_own_obstacle_not_a_verdict(): + result = _review( + status="blocked", + summary="The pull request head could not be resolved from any SCM mirror.", + blockers=["Gitea API is unreachable from the review worktree."], + ) + + assert goal.unfinished_result_reason(result, role=goal.REVIEW_ROLE) is None + assert goal.unfinished_result_reason(result) is None + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, result, open_request=_no_judge + ) + assert accepted is False + assert "rather than a finished task" in reason + + +def test_conflicting_role_directives_hold_the_card_to_implementation(): + card = REVIEW_CARD.replace( + "## Body\n", + "## Body\nHermes-Task-Role: review\nHermes-Task-Role: implementation\n", + ) + + assert goal.task_role(card, _review()) == (goal.IMPLEMENTATION_ROLE, "directive") + + +def test_incomplete_review_summary_is_bounded_before_it_leaves_the_judge(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review(status="incomplete", summary="pending\n" + "y" * 4000), + open_request=_no_judge, + ) + + assert accepted is False + assert reason.startswith(goal.READ_ONLY_GUARD) + assert len(reason) <= goal.JUDGE_REASON_LIMIT + + +def test_review_claiming_completion_with_blockers_is_rejected(): + accepted, reason = goal.judge_goal_completion( + REVIEW_CARD, + _review(blockers=["the diff could not be resolved"]), + open_request=_no_judge, + ) + + assert accepted is False + assert "worker reported blockers while claiming completion" in reason + + +def test_review_prose_about_the_reviewed_artifact_is_not_unfinished_work(): + result = _review( + summary=( + "Independent read-only review. Verdict: BLOCK. The reviewed branch's CI " + "build is still running and its migration remains pending, which is why " + "the change is unfit to ship." + ), + tests_run=["upstream pipeline: build is still running on the reviewed head"], + ) + + assert goal.unfinished_result_reason(result, role=goal.REVIEW_ROLE) is None + accepted, _reason = goal.judge_goal_completion( + REVIEW_CARD, + result, + open_request=_no_judge, + ) + assert accepted is True + + +def test_implementation_prose_about_unfinished_work_is_still_rejected(): + result = _implementation(summary="The broad rerun remains active before the push.") + + assert goal.unfinished_result_reason(result) + assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) + + +class JudgeResponse: + def __init__(self, verdict: str, reason: str): + self.body = json.dumps( + {"choices": [{"message": {"content": json.dumps( + {"verdict": verdict, "reason": reason} + )}}]} + ).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return self.body + + +def test_implementation_card_keeps_the_fail_closed_model_judge(): + observed = {} + + def request(req, timeout): + observed["payload"] = json.loads(req.data) + observed["timeout"] = timeout + return JudgeResponse("continue", "the remote head was never verified") + + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(), + open_request=request, + ) + + assert accepted is False + assert reason == "the remote head was never verified" + role = json.loads(observed["payload"]["messages"][1]["content"])["task_role"] + assert role == goal.IMPLEMENTATION_ROLE + assert observed["timeout"] == 120 + + +def test_implementation_completion_still_finalizes_on_an_accepting_judge(): + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(), + open_request=lambda *_a, **_k: JudgeResponse("complete", "every criterion has evidence"), + ) + + assert accepted is True + assert reason == "every criterion has evidence" + + +def test_ambiguous_card_without_a_verdict_report_stays_on_the_model_judge(): + ambiguous = """# Kanban task t_mixed: Review and repair the pool + +## Body +Review the pool, then repair what you find. Do not modify unrelated files. +Return a verdict and open a draft PR with the repairs. +""" + + accepted, _reason = goal.judge_goal_completion( + ambiguous, + _review(), + open_request=lambda *_a, **_k: JudgeResponse("continue", "the repairs are missing"), + ) + + assert goal.task_role(ambiguous, _review())[0] == goal.IMPLEMENTATION_ROLE + assert accepted is False + + +def test_judge_reason_is_bounded_sanitized_and_secret_free(): + leaky = "line one\r\n\x07 token ghp_" + "A" * 24 + " " + "x" * 900 + + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(), + open_request=lambda *_a, **_k: JudgeResponse("continue", leaky), + ) + + assert accepted is False + assert len(reason) <= goal.JUDGE_REASON_LIMIT + assert "ghp_" not in reason + assert "[redacted]" in reason + assert "\n" not in reason and "\r" not in reason and "\x07" not in reason + + +def test_unavailable_judge_still_fails_closed_for_implementation_cards(): + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(), + open_request=lambda *_a, **_k: JudgeResponse("unknown", "bad"), + ) + + assert accepted is False + assert "local completion judge unavailable" in reason + + +def test_bare_uppercase_verdict_token_is_accepted_as_a_last_resort(): + result = _review( + summary=( + "Independent read-only review of PR #18 at head 2000252 concludes BLOCK " + "because five reproducible P0 defects fire on the first coordinator tick." + ), + ) + + assert goal.review_verdict(result) == ("BLOCK", None) + accepted, _reason = goal.judge_goal_completion( + REVIEW_CARD, result, open_request=_no_judge + ) + assert accepted is True + + +def test_unknown_role_directive_falls_through_to_the_next_contract_surface(): + card = IMPLEMENTATION_CARD.replace( + "## Body\n", + "## Body\nHermes-Task-Role: gardener\nHermes-Expected-Output: verdict\n", + ) + + assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "directive") + + +def test_unrecognized_directives_leave_the_role_to_inference(): + card = REVIEW_CARD.replace("## Body\n", "## Body\nHermes-Task-Role: gardener\n") + + assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "inferred") + + +def test_malformed_evidence_fields_are_read_without_raising(): + assert goal._strings("not-a-list") == [] + result = _implementation(tests_run="pytest -q", summary="Everything shipped.") + + assert goal.unfinished_result_reason(result) is None + + +def test_implementation_rejection_reason_carries_no_read_only_guard(): + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(blockers=["the remote was unreachable"]), + open_request=_no_judge, + ) + + assert accepted is False + assert reason == "worker reported blockers while claiming completion" + + +def test_oversized_objectives_and_empty_reasons_stay_bounded(): + compacted = goal._bounded("a" * 9000, 6000) + + assert len(compacted) == 6000 + assert "[goal evidence compacted]" in compacted + assert goal.sanitize_reason(" ") == "local judge supplied no reason" + + +def test_judging_never_mutates_the_reported_result(): + result = _review() + snapshot = json.dumps(result, sort_keys=True) + + goal.judge_goal_completion(REVIEW_CARD, result, open_request=_no_judge) + + assert json.dumps(result, sort_keys=True) == snapshot diff --git a/testing/tests/test_hermes_cli_review_lane.py b/testing/tests/test_hermes_cli_review_lane.py new file mode 100644 index 00000000..556b4634 --- /dev/null +++ b/testing/tests/test_hermes_cli_review_lane.py @@ -0,0 +1,427 @@ +"""Lane-level goal-loop behaviour for review and implementation cards.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +# The single-claim loop lives in the runner today and in ``cli_lane_execution`` +# once the lane modules are split. Patch whichever module owns it so this file +# describes lane behaviour rather than one file layout. +execution = sys.modules.get("cli_lane_execution", lanes) +goal = sys.modules["cli_lane_goal"] + +REVIEW_CARD = """# Kanban task t_review: Independent read-only review of the worker pool + +## Body +Perform a fresh independent read-only release review of the pool implementation. +Do not edit files, comment, commit, push, merge, publish, or deploy. +Return strict SHIP or BLOCK with exact file/line and reproducible inputs. +""" +IMPLEMENTATION_CARD = """# Kanban task t_impl: Repair the pool blockers + +## Body +Repair every reported blocker, run the focused suites, and open a draft PR. +Return exact pushed SHA and evidence. +""" +BLOCK_REVIEW = { + "status": "completed", + "summary": ( + "Independent read-only review of PR #18 at head 2000252. Verdict: BLOCK. " + "Five reproducible P0 defects fire on the coordinator's first tick and the " + "review worktree was left pristine." + ), + "changed_files": [], + "tests_run": ["pytest testing/tests/test_pool.py: 82 passed"], + "artifacts": [], + "findings": ["P0-1 BLOCKER - coordinator.py:248 compares int to str run ids."], + "blockers": [], +} + + +class _Connection: + def close(self) -> None: + return None + + +class _Lane: + """One instrumented direct-CLI-lane claim over a fake Kanban board.""" + + def __init__(self, tmp_path: Path, monkeypatch, *, card: str, task, reports): + self.calls: list[tuple[str, dict]] = [] + self.comments: list[str] = [] + self.prompts: list[str] = [] + self.routes: list[tuple[str, dict]] = [] + self.heartbeats: list[tuple[str, object]] = [] + self.reports = list(reports) + self.task = task + self.state_root = tmp_path / "cli-lanes" + self.workspace = tmp_path + self.db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board=None, **_kwargs: _Connection(), + get_task=lambda _conn, _task_id: self.task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_card"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: card, + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + self.heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda _conn, _task_id, _author, body: self.comments.append(body), + complete_task=self._complete_task, + block_task=self._block_task, + reclaim_task=self._reclaim_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=self.db)) + # The lane derives durable journal identity from ``STATE_ROOT``, so + # relocate the root itself rather than stubbing ``state_path``. + for module in list(sys.modules.values()): + if getattr(module, "__name__", "").startswith("cli_lane") and hasattr( + module, "STATE_ROOT" + ): + monkeypatch.setattr(module, "STATE_ROOT", self.state_root) + monkeypatch.setattr(execution, "fresh_unavailable_provider", lambda: None) + monkeypatch.setattr(execution, "select_route", self._select_route) + monkeypatch.setattr(execution, "run_provider", self._run_provider) + + def _complete_task( + self, + _conn, + task_id, + *, + result, + summary, + metadata, + expected_run_id=None, + replay_ended_run_id=None, + ): + self.calls.append( + ( + "complete", + { + "task_id": task_id, + "result": result, + "summary": summary, + "metadata": metadata, + "expected_run_id": expected_run_id or replay_ended_run_id, + }, + ) + ) + self.task = SimpleNamespace( + **{ + **vars(self.task), + "status": "done", + "result": result, + "completed_run_id": expected_run_id or replay_ended_run_id, + } + ) + return True + + def _block_task(self, _conn, task_id, *, reason, kind, expected_run_id=None): + self.calls.append( + ( + "block", + { + "task_id": task_id, + "reason": reason, + "kind": kind, + "expected_run_id": expected_run_id, + }, + ) + ) + return True + + def _reclaim_task(self, _conn, _task_id, *, reason=None, expected_run_id=None): + return True + + @property + def state_file(self) -> Path: + return self.state_root / "titan-iac/t_card.json" + + @staticmethod + def _route(provider: str, effort: str = "xhigh"): + return lanes.Route( + provider, + f"{provider}-model", + effort, + f"{provider}-{effort}", + "switchyard", + "routed", + 1, + (), + ) + + def _select_route(self, _prompt, assignee, **kwargs): + self.routes.append((assignee, kwargs)) + if assignee.startswith("cli-codex"): + return self._route("codex") + return self._route("claude") + + def _run_provider(self, *args, **_kwargs): + self.prompts.append(args[1]) + args[6]("working") + return self.reports.pop(0) + + @property + def terminal(self) -> tuple[str, dict]: + assert len(self.calls) == 1, self.calls + return self.calls[0] + + @property + def state(self) -> dict: + return json.loads(self.state_file.read_text(encoding="utf-8")) + + def rejections(self) -> list[str]: + return [item for item in self.comments if "Goal completion rejected" in item] + + +def _task(**overrides): + value = { + "id": "t_card", + "status": "running", + "current_run_id": 23, + "completed_run_id": None, + "result": None, + "assignee": "cli-claude-xhigh", + "max_runtime_seconds": 600, + "goal_mode": True, + "goal_max_turns": 8, + } + value.update(overrides) + return SimpleNamespace(**value) + + +def _result(**overrides): + value = dict(BLOCK_REVIEW) + value.update(overrides) + return lanes.ProcessResult(0, "review turn", value, False) + + +def _no_judge(*_args, **_kwargs): + raise AssertionError("the model judge must not be consulted for a review card") + + +def test_completed_block_review_finalizes_on_its_first_turn(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(), + reports=[_result()], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["expected_run_id"] == 23 + assert json.loads(kwargs["result"])["findings"] == BLOCK_REVIEW["findings"] + assert kwargs["metadata"]["goal_turn"] == 1 + assert "BLOCK verdict with 1 finding(s)" in kwargs["metadata"]["goal_judge_reason"] + assert lane.rejections() == [] + assert len(lane.prompts) == 1 + assert lane.reports == [] + + +def test_review_verdict_reaches_kanban_without_hidden_mutation(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(), + reports=[_result()], + ) + + execution.execute_claim("titan-iac", "t_card") + + _action, kwargs = lane.terminal + stored = json.loads(kwargs["result"]) + assert stored["status"] == "completed" + assert stored["summary"] == BLOCK_REVIEW["summary"] + assert stored["blockers"] == [] + assert kwargs["summary"] == BLOCK_REVIEW["summary"] + assert kwargs["metadata"]["findings"] == BLOCK_REVIEW["findings"] + assert len(kwargs["metadata"]["goal_judge_reason"]) <= goal.JUDGE_REASON_LIMIT + + +def test_reviewer_provider_fallback_still_finalizes_one_verdict(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(assignee="cli-auto"), + reports=[ + lanes.ProcessResult(1, "usage limit reached", None, True), + _result(), + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["metadata"]["provider"] == "codex" + assert kwargs["metadata"]["goal_turn"] == 1 + assert any("Provider fallback: claude -> codex" in item for item in lane.comments) + assert lane.rejections() == [] + assert len(lane.prompts) == 2 + + +def test_review_restart_resumes_the_next_turn_and_completes_once(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(), + reports=[_result()], + ) + lane.state_file.parent.mkdir(parents=True, exist_ok=True) + lane.state_file.write_text( + json.dumps({"goal_turn": 3, "current_route": {"provider": "codex"}}), + encoding="utf-8", + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["metadata"]["goal_turn"] == 4 + assert lane.state["goal_turn"] == 4 + assert any("Restart-time provider change" in item for item in lane.comments) + + +def test_malformed_review_is_resumed_without_demanding_an_edit(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(goal_max_turns=2), + reports=[ + _result(summary="Review done.", findings=[]), + _result(), + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["metadata"]["goal_turn"] == 2 + assert len(lane.rejections()) == 1 + assert goal.READ_ONLY_GUARD in lane.rejections()[0] + assert "do not modify the reviewed implementation" in lane.prompts[1] + assert "Goal-loop continuation" in lane.prompts[1] + + +def test_exhausted_review_budget_blocks_with_a_read_only_reason(tmp_path, monkeypatch): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(goal_max_turns=1), + reports=[_result(summary="Review done.", findings=[])], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "block" + assert kwargs["expected_run_id"] == 23 + assert goal.READ_ONLY_GUARD in kwargs["reason"] + + +def test_implementation_goal_loop_is_preserved(tmp_path, monkeypatch): + verdicts = iter([(False, "the remote head was never verified"), (True, "all criteria hold")]) + contexts: list[str] = [] + + def judge(objective, *_args, **_kwargs): + contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr(goal, "judge_goal_completion", judge) + implementation = { + "status": "completed", + "summary": "Focused tests passed and the branch was pushed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + } + lane = _Lane( + tmp_path, + monkeypatch, + card=IMPLEMENTATION_CARD, + task=_task(assignee="cli-auto", goal_max_turns=3), + reports=[ + lanes.ProcessResult(0, "turn one", dict(implementation), False), + lanes.ProcessResult(0, "turn two", dict(implementation), False), + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["metadata"]["goal_turn"] == 2 + assert len(lane.rejections()) == 1 + assert "the remote head was never verified" in lane.rejections()[0] + assert "prior rejected reports" in contexts[1] + + +def test_incomplete_implementation_never_finalizes(tmp_path, monkeypatch): + lane = _Lane( + tmp_path, + monkeypatch, + card=IMPLEMENTATION_CARD, + task=_task(assignee="cli-auto", goal_max_turns=1), + reports=[ + lanes.ProcessResult( + 0, + "turn one", + { + "status": "incomplete", + "summary": "The focused suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ) + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "block" + assert "The focused suite is still running." in kwargs["reason"] From ed43dbaa7f094e1841aa760862e80015deb9d80c Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 15:33:56 +0000 Subject: [PATCH 17/46] hermes: split AI usage exporter engines --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/kustomization.yaml | 2 + services/hermes/scripts/ai_usage_codex.py | 138 +++++++++ services/hermes/scripts/ai_usage_exporter.py | 244 +++++----------- services/hermes/scripts/ai_usage_http.py | 49 ++++ testing/tests/test_hermes_ai_usage_codex.py | 255 ++++++++++++++++ .../tests/test_hermes_ai_usage_exporter.py | 163 +++-------- .../test_hermes_ai_usage_exporter_coverage.py | 275 ++++++++++++++++++ testing/tests/test_hermes_auto_router.py | 140 +++++---- 9 files changed, 903 insertions(+), 365 deletions(-) create mode 100644 services/hermes/scripts/ai_usage_codex.py create mode 100644 services/hermes/scripts/ai_usage_http.py create mode 100644 testing/tests/test_hermes_ai_usage_codex.py create mode 100644 testing/tests/test_hermes_ai_usage_exporter_coverage.py diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 6cb07c34..84c0bcd2 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-ai-usage-poller-v1" + ai.bstein.dev/config-rev: "20260817-ai-usage-exporter-split-v2" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..f7699e51 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -58,7 +58,9 @@ configMapGenerator: - name: hermes-coordinator namespace: hermes files: + - ai_usage_codex.py=scripts/ai_usage_codex.py - ai_usage_exporter.py=scripts/ai_usage_exporter.py + - ai_usage_http.py=scripts/ai_usage_http.py - claude=scripts/claude - claude_command_policy.py=scripts/claude_command_policy.py - cli_lane_goal.py=scripts/cli_lane_goal.py diff --git a/services/hermes/scripts/ai_usage_codex.py b/services/hermes/scripts/ai_usage_codex.py new file mode 100644 index 00000000..6214cbf5 --- /dev/null +++ b/services/hermes/scripts/ai_usage_codex.py @@ -0,0 +1,138 @@ +"""Bounded Codex app-server query and child-process cleanup.""" + +from __future__ import annotations + +import json +import os +import selectors +import subprocess +import time +from typing import Any + + +CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") +CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") +CODEX_CLEANUP_TIMEOUT_SECONDS = 5 + + +def _close_stream(stream: Any) -> bool: + """Close a subprocess pipe without allowing cleanup errors to escape.""" + if stream is None: + return True + try: + stream.close() + except Exception: + return False + return True + + +def _cleanup_codex_process( + process: subprocess.Popen[str], selector: selectors.BaseSelector | None +) -> bool: + """Stop and reap a Codex child within two fixed waits, then close its pipes.""" + cleanup_ok = True + if selector is not None: + try: + selector.close() + except Exception: + cleanup_ok = False + + # Closing the request pipe first also gives a responsive app-server an EOF. + cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok + try: + process.terminate() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + + needs_kill = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + needs_kill = True + except Exception: + cleanup_ok = False + needs_kill = True + + if needs_kill: + try: + process.kill() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except Exception: + cleanup_ok = False + + cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok + cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok + return cleanup_ok + + +def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: + """Read Codex account quota and usage through its structured app-server protocol.""" + process = subprocess.Popen( + [CODEX_BIN, "app-server", "--stdio"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + env={**os.environ, "CODEX_HOME": CODEX_HOME}, + ) + requests = ( + { + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "atlas-ai-usage-exporter", + "title": "Atlas AI Usage Exporter", + "version": "1.0.0", + }, + "capabilities": {"experimentalApi": True}, + }, + }, + {"method": "initialized", "params": {}}, + {"id": 2, "method": "account/rateLimits/read", "params": None}, + {"id": 3, "method": "account/usage/read", "params": None}, + ) + selector: selectors.BaseSelector | None = None + rate_result: dict[str, Any] + usage_result: dict[str, Any] + try: + if process.stdin is None or process.stdout is None: + raise RuntimeError("Codex app-server pipes are unavailable") + for message in requests: + process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + process.stdin.flush() + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + responses: dict[int, dict[str, Any]] = {} + deadline = time.monotonic() + timeout + while len(responses) < 3 and time.monotonic() < deadline: + for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))): + line = key.fileobj.readline() + if not line: + continue + message = json.loads(line) + if message.get("id") in (1, 2, 3): + responses[int(message["id"])] = message + for response_id in (1, 2, 3): + response = responses.get(response_id) + if ( + not response + or "error" in response + or not isinstance(response.get("result"), dict) + ): + raise RuntimeError(f"Codex app-server response {response_id} failed") + rate_result = responses[2]["result"] + usage_result = responses[3]["result"] + finally: + cleanup_ok = _cleanup_codex_process(process, selector) + if not cleanup_ok: + raise RuntimeError("Codex app-server cleanup failed") + return rate_result, usage_result diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index 16be50a6..b0db51fb 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -5,20 +5,18 @@ from __future__ import annotations import json import os -import selectors -import subprocess import threading import time from dataclasses import dataclass, field from datetime import UTC, date, datetime, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any from urllib.request import Request, urlopen +import ai_usage_codex as codex_query +import ai_usage_http as http_engine + -CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") -CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") CLAUDE_CREDENTIALS = Path( os.environ.get( "ATLAS_AI_CLAUDE_CREDENTIALS", @@ -48,7 +46,6 @@ METRIC_HELP = { "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.", "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.", } -CODEX_CLEANUP_TIMEOUT_SECONDS = 5 POLL_STARTUP_GRACE_SECONDS = 45 POLL_PROGRESS_BUDGET_SECONDS = 60 @@ -78,7 +75,9 @@ def _number(value: Any) -> float | None: if isinstance(value, bool) or not isinstance(value, (int, float)): return None converted = float(value) - return converted if converted == converted and abs(converted) != float("inf") else None + return ( + converted if converted == converted and abs(converted) != float("inf") else None + ) def _timestamp(value: Any) -> float | None: @@ -115,7 +114,10 @@ def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str: def parse_codex_payloads( - rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None + rate_response: dict[str, Any], + usage_response: dict[str, Any], + *, + today: date | None = None, ) -> list[Sample]: """Convert structured Codex app-server responses into bounded metrics.""" samples: list[Sample] = [] @@ -143,15 +145,23 @@ def parse_codex_payloads( samples.extend( ( Sample("atlas_ai_quota_used_percent", labels, used), - Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)), + Sample( + "atlas_ai_quota_remaining_percent", labels, max(0, 100 - used) + ), ) ) reset = _timestamp(raw_window.get("resetsAt")) duration = _number(raw_window.get("windowDurationMins")) if reset is not None: - samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + samples.append( + Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset) + ) if duration is not None: - samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60)) + samples.append( + Sample( + "atlas_ai_quota_window_duration_seconds", labels, duration * 60 + ) + ) summary = usage_response.get("summary") if isinstance(summary, dict): @@ -192,8 +202,16 @@ def parse_codex_payloads( anchor_day = max((start for start, _ in parsed_daily), default=current_day) for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)): earliest = anchor_day - timedelta(days=days - 1) - value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day) - samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value)) + value = sum( + tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day + ) + samples.append( + Sample( + "atlas_ai_account_tokens", + {"provider": "openai", "period": period}, + value, + ) + ) return samples @@ -216,7 +234,9 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: ) reset = _timestamp(window.get("resets_at")) if reset is not None: - samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + samples.append( + Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset) + ) extra = payload.get("extra_usage") if isinstance(extra, dict): enabled = extra.get("is_enabled") @@ -231,123 +251,7 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: return samples -def _close_stream(stream: Any) -> bool: - """Close a subprocess pipe without allowing cleanup errors to escape.""" - if stream is None: - return True - try: - stream.close() - except Exception: - return False - return True - - -def _cleanup_codex_process( - process: subprocess.Popen[str], selector: selectors.BaseSelector | None -) -> bool: - """Stop and reap a Codex child within two fixed waits, then close its pipes.""" - cleanup_ok = True - if selector is not None: - try: - selector.close() - except Exception: - cleanup_ok = False - - # Closing the request pipe first also gives a responsive app-server an EOF. - cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok - try: - process.terminate() - except ProcessLookupError: - pass - except Exception: - cleanup_ok = False - - needs_kill = False - try: - process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - needs_kill = True - except Exception: - cleanup_ok = False - needs_kill = True - - if needs_kill: - try: - process.kill() - except ProcessLookupError: - pass - except Exception: - cleanup_ok = False - try: - process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) - except Exception: - cleanup_ok = False - - cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok - cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok - return cleanup_ok - - -def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: - """Read Codex account quota and usage through its structured app-server protocol.""" - process = subprocess.Popen( - [CODEX_BIN, "app-server", "--stdio"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - bufsize=1, - env={**os.environ, "CODEX_HOME": CODEX_HOME}, - ) - requests = ( - { - "id": 1, - "method": "initialize", - "params": { - "clientInfo": { - "name": "atlas-ai-usage-exporter", - "title": "Atlas AI Usage Exporter", - "version": "1.0.0", - }, - "capabilities": {"experimentalApi": True}, - }, - }, - {"method": "initialized", "params": {}}, - {"id": 2, "method": "account/rateLimits/read", "params": None}, - {"id": 3, "method": "account/usage/read", "params": None}, - ) - selector: selectors.BaseSelector | None = None - rate_result: dict[str, Any] - usage_result: dict[str, Any] - try: - if process.stdin is None or process.stdout is None: - raise RuntimeError("Codex app-server pipes are unavailable") - for message in requests: - process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") - process.stdin.flush() - selector = selectors.DefaultSelector() - selector.register(process.stdout, selectors.EVENT_READ) - responses: dict[int, dict[str, Any]] = {} - deadline = time.monotonic() + timeout - while len(responses) < 3 and time.monotonic() < deadline: - for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))): - line = key.fileobj.readline() - if not line: - continue - message = json.loads(line) - if message.get("id") in (1, 2, 3): - responses[int(message["id"])] = message - for response_id in (1, 2, 3): - response = responses.get(response_id) - if not response or "error" in response or not isinstance(response.get("result"), dict): - raise RuntimeError(f"Codex app-server response {response_id} failed") - rate_result = responses[2]["result"] - usage_result = responses[3]["result"] - finally: - cleanup_ok = _cleanup_codex_process(process, selector) - if not cleanup_ok: - raise RuntimeError("Codex app-server cleanup failed") - return rate_result, usage_result +query_codex = codex_query.query_codex def query_claude() -> dict[str, Any]: @@ -393,7 +297,9 @@ class Collector: ) -> None: """Record a failed attempt while retaining the provider's last good samples.""" attempted_at = time.time() if started is None else started - duration_started = time.monotonic() if monotonic_started is None else monotonic_started + duration_started = ( + time.monotonic() if monotonic_started is None else monotonic_started + ) with self._lock: state = self._providers[provider] state.last_attempt = attempted_at @@ -412,7 +318,10 @@ class Collector: else: raise ValueError("unknown provider") except Exception as error: - print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True) + print( + f"{provider} quota collection deferred: {type(error).__name__}", + flush=True, + ) self.record_failure( provider, started=started, @@ -440,10 +349,24 @@ class Collector: samples.extend(state.samples) samples.extend( ( - Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)), - Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt), - Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success), - Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration), + Sample( + "atlas_ai_quota_fetch_success", + labels, + float(state.fetch_success), + ), + Sample( + "atlas_ai_quota_last_attempt_timestamp_seconds", + labels, + state.last_attempt, + ), + Sample( + "atlas_ai_quota_last_success_timestamp_seconds", + labels, + state.last_success, + ), + Sample( + "atlas_ai_quota_fetch_duration_seconds", labels, state.duration + ), ) ) lines: list[str] = [] @@ -525,7 +448,9 @@ class PollingEngine: try: self.poll_once() except Exception as error: - print(f"quota polling cycle deferred: {type(error).__name__}", flush=True) + print( + f"quota polling cycle deferred: {type(error).__name__}", flush=True + ) time.sleep(self.interval) def start(self) -> None: @@ -552,47 +477,8 @@ class PollingEngine: return now - last_progress <= self.progress_timeout -def make_handler(collector: Collector, poller: PollingEngine) -> type[BaseHTTPRequestHandler]: - """Build an HTTP handler bound to one collector and polling engine.""" - - class Handler(BaseHTTPRequestHandler): - def _respond(self, status: int, payload: bytes, content_type: str) -> None: - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def do_GET(self) -> None: # noqa: N802 - try: - if self.path == "/metrics": - self._respond( - 200, - collector.render(), - "text/plain; version=0.0.4", - ) - elif self.path == "/healthz": - healthy = poller.is_healthy() - self._respond( - 200 if healthy else 503, - b"ok\n" if healthy else b"poller unhealthy\n", - "text/plain; charset=utf-8", - ) - else: - self.send_error(404) - except (BrokenPipeError, ConnectionResetError): - return - - def log_message(self, _format: str, *_args: object) -> None: - return - - return Handler - - -class Server(ThreadingHTTPServer): - """Threaded metrics server that does not retain disconnected clients.""" - - daemon_threads = True +make_handler = http_engine.make_handler +Server = http_engine.Server def main() -> int: diff --git a/services/hermes/scripts/ai_usage_http.py b/services/hermes/scripts/ai_usage_http.py new file mode 100644 index 00000000..8558740f --- /dev/null +++ b/services/hermes/scripts/ai_usage_http.py @@ -0,0 +1,49 @@ +"""Disconnect-safe HTTP serving for the AI usage exporter.""" + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + + +def make_handler(collector: Any, poller: Any) -> type[BaseHTTPRequestHandler]: + """Build an HTTP handler bound to one collector and polling engine.""" + + class Handler(BaseHTTPRequestHandler): + def _respond(self, status: int, payload: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: # noqa: N802 + try: + if self.path == "/metrics": + self._respond( + 200, + collector.render(), + "text/plain; version=0.0.4", + ) + elif self.path == "/healthz": + healthy = poller.is_healthy() + self._respond( + 200 if healthy else 503, + b"ok\n" if healthy else b"poller unhealthy\n", + "text/plain; charset=utf-8", + ) + else: + self.send_error(404) + except (BrokenPipeError, ConnectionResetError): + return + + def log_message(self, _format: str, *_args: object) -> None: + return + + return Handler + + +class Server(ThreadingHTTPServer): + """Threaded metrics server that does not retain disconnected clients.""" + + daemon_threads = True diff --git a/testing/tests/test_hermes_ai_usage_codex.py b/testing/tests/test_hermes_ai_usage_codex.py new file mode 100644 index 00000000..a77525bf --- /dev/null +++ b/testing/tests/test_hermes_ai_usage_codex.py @@ -0,0 +1,255 @@ +"""Adversarial tests for bounded Codex quota-query process handling.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from io import StringIO +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "services/hermes/scripts/ai_usage_codex.py" + + +def load_module(): + """Load the standalone Codex query helper as a fresh test module.""" + name = "ai_usage_codex_tested" + spec = importlib.util.spec_from_file_location(name, SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): + mod = load_module() + request_log = tmp_path / "requests.jsonl" + mock = tmp_path / "codex-mock" + mock.write_text( + """#!/usr/bin/env python3 +import json, os, sys +messages = [json.loads(sys.stdin.readline()) for _ in range(4)] +with open(os.environ['REQUEST_LOG'], 'w') as handle: + for message in messages: + handle.write(json.dumps(message) + '\\n') +responses = { + 1: {}, + 2: {'rateLimits': {'primary': {'usedPercent': 10}}}, + 3: {'summary': {}, 'dailyUsageBuckets': []}, +} +for response_id, result in responses.items(): + print(json.dumps({'id': response_id, 'result': result}), flush=True) +""" + ) + mock.chmod(0o755) + monkeypatch.setattr(mod, "CODEX_BIN", str(mock)) + monkeypatch.setenv("REQUEST_LOG", str(request_log)) + + rate, usage = mod.query_codex(timeout=5) + requests = [json.loads(line) for line in request_log.read_text().splitlines()] + + assert rate["rateLimits"]["primary"]["usedPercent"] == 10 + assert usage["summary"] == {} + assert [request["method"] for request in requests] == [ + "initialize", + "initialized", + "account/rateLimits/read", + "account/usage/read", + ] + assert all("/status" not in json.dumps(request) for request in requests) + + +def test_codex_cleanup_is_bounded_when_process_cannot_be_reaped(monkeypatch): + mod = load_module() + processes = [] + selectors = [] + + class StubbornProcess: + def __init__(self, *_args, **_kwargs): + self.stdin = StringIO() + self.stdout = StringIO( + "\n".join( + json.dumps({"id": response_id, "result": result}) + for response_id, result in ( + (1, {}), + (2, {"rateLimits": {}}), + (3, {"summary": {}, "dailyUsageBuckets": []}), + ) + ) + + "\n" + ) + self.stderr = None + self.terminate_calls = 0 + self.kill_calls = 0 + self.wait_timeouts = [] + processes.append(self) + + def terminate(self): + self.terminate_calls += 1 + + def kill(self): + self.kill_calls += 1 + + def wait(self, timeout): + self.wait_timeouts.append(timeout) + raise subprocess.TimeoutExpired("codex-mock", timeout) + + class TrackingSelector: + def __init__(self): + self.fileobj = None + self.closed = False + selectors.append(self) + + def register(self, fileobj, _events): + self.fileobj = fileobj + + def select(self, _timeout): + return [(SimpleNamespace(fileobj=self.fileobj), None)] + + def close(self): + self.closed = True + + monkeypatch.setattr(mod.subprocess, "Popen", StubbornProcess) + monkeypatch.setattr(mod.selectors, "DefaultSelector", TrackingSelector) + + with pytest.raises(RuntimeError, match="^Codex app-server cleanup failed$"): + mod.query_codex(timeout=1) + + process = processes[0] + assert process.terminate_calls == 1 + assert process.kill_calls == 1 + assert process.wait_timeouts == [5, 5] + assert process.stdin.closed + assert process.stdout.closed + assert selectors[0].closed + + +def test_cleanup_contains_all_pipe_and_process_api_failures(): + mod = load_module() + + class BrokenStream: + def close(self): + raise OSError("private pipe detail") + + class BrokenSelector: + def close(self): + raise OSError("private selector detail") + + class BrokenProcess: + stdin = BrokenStream() + stdout = BrokenStream() + stderr = BrokenStream() + + def __init__(self): + self.wait_calls = 0 + + def terminate(self): + raise OSError("private terminate detail") + + def wait(self, timeout): + assert timeout == mod.CODEX_CLEANUP_TIMEOUT_SECONDS + self.wait_calls += 1 + raise OSError("private wait detail") + + def kill(self): + raise OSError("private kill detail") + + assert mod._close_stream(None) + assert not mod._close_stream(BrokenStream()) + assert not mod._cleanup_codex_process(BrokenProcess(), BrokenSelector()) + + +def test_cleanup_accepts_already_exited_process_during_terminate_and_kill(): + mod = load_module() + + class AlreadyExited: + stdin = None + stdout = None + stderr = None + + def __init__(self, *, timeout_first): + self.timeout_first = timeout_first + self.wait_calls = 0 + + def terminate(self): + raise ProcessLookupError + + def wait(self, timeout): + self.wait_calls += 1 + if self.timeout_first and self.wait_calls == 1: + raise subprocess.TimeoutExpired("codex", timeout) + + def kill(self): + raise ProcessLookupError + + assert mod._cleanup_codex_process(AlreadyExited(timeout_first=False), None) + assert mod._cleanup_codex_process(AlreadyExited(timeout_first=True), None) + + +def test_query_rejects_missing_pipes_and_structured_errors(monkeypatch): + mod = load_module() + + class Process: + stderr = None + + def __init__(self, *_args, **_kwargs): + self.stdin = None + self.stdout = None + + def terminate(self): + return None + + def wait(self, timeout): + return 0 + + monkeypatch.setattr(mod.subprocess, "Popen", Process) + with pytest.raises(RuntimeError, match="pipes are unavailable"): + mod.query_codex(timeout=0) + + class InvalidProcess(Process): + def __init__(self, *_args, **_kwargs): + self.stdin = StringIO() + self.stdout = SequencedOutput() + + class SequencedOutput: + def __init__(self): + self.lines = iter( + [ + "", + json.dumps({"id": 99, "result": {}}) + "\n", + json.dumps({"id": 1, "result": {}}) + "\n", + json.dumps({"id": 2, "error": {"message": "secret"}}) + "\n", + json.dumps({"id": 3, "result": {}}) + "\n", + ] + ) + + def readline(self): + return next(self.lines) + + def close(self): + return None + + class Selector: + def register(self, fileobj, _events): + self.fileobj = fileobj + + def select(self, _timeout): + return [(SimpleNamespace(fileobj=self.fileobj), None)] + + def close(self): + return None + + monkeypatch.setattr(mod.subprocess, "Popen", InvalidProcess) + monkeypatch.setattr(mod.selectors, "DefaultSelector", Selector) + with pytest.raises( + RuntimeError, match="^Codex app-server response 2 failed$" + ) as caught: + mod.query_codex(timeout=1) + assert "secret" not in str(caught.value) diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index 9fe54d72..c33654dc 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -3,13 +3,10 @@ from __future__ import annotations import importlib.util -import json -import subprocess import sys from datetime import date -from io import BytesIO, StringIO +from io import BytesIO from pathlib import Path -from types import SimpleNamespace ROOT = Path(__file__).resolve().parents[2] @@ -18,6 +15,9 @@ SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" def load_module(): """Load the standalone exporter script as a test module.""" + script_directory = str(SCRIPT.parent) + if script_directory not in sys.path: + sys.path.insert(0, script_directory) spec = importlib.util.spec_from_file_location("ai_usage_exporter", SCRIPT) module = importlib.util.module_from_spec(spec) assert spec and spec.loader @@ -57,7 +57,9 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods(): } samples = mod.parse_codex_payloads(rate, usage, today=date(2026, 8, 16)) - values = {(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples} + values = { + (item.name, tuple(sorted(item.labels.items()))): item.value for item in samples + } codex_labels = ( ("limit", "codex"), @@ -72,18 +74,24 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods(): assert values[("atlas_ai_quota_used_percent", codex_labels)] == 41 assert values[("atlas_ai_quota_remaining_percent", codex_labels)] == 59 assert values[("atlas_ai_quota_used_percent", spark_labels)] == 0 - assert values[ - ( - "atlas_ai_account_tokens", - (("period", "latest_day"), ("provider", "openai")), - ) - ] == 100 - assert values[ - ( - "atlas_ai_account_tokens", - (("period", "seven_day"), ("provider", "openai")), - ) - ] == 150 + assert ( + values[ + ( + "atlas_ai_account_tokens", + (("period", "latest_day"), ("provider", "openai")), + ) + ] + == 100 + ) + assert ( + values[ + ( + "atlas_ai_account_tokens", + (("period", "seven_day"), ("provider", "openai")), + ) + ] + == 150 + ) def test_claude_payload_uses_only_supported_quota_fields(): @@ -134,120 +142,21 @@ def test_render_reports_failure_and_freshness_without_logging_credentials(): rendered = collector.render().decode() assert 'atlas_ai_quota_fetch_success{provider="openai"} 0' in rendered - assert 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' in rendered - assert 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered + assert ( + 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' + in rendered + ) + assert ( + 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' + in rendered + ) assert "accessToken" not in rendered assert "refreshToken" not in rendered -def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): - mod = load_module() - request_log = tmp_path / "requests.jsonl" - mock = tmp_path / "codex-mock" - mock.write_text( - """#!/usr/bin/env python3 -import json, os, sys -messages = [json.loads(sys.stdin.readline()) for _ in range(4)] -with open(os.environ['REQUEST_LOG'], 'w') as handle: - for message in messages: - handle.write(json.dumps(message) + '\\n') -responses = { - 1: {}, - 2: {'rateLimits': {'primary': {'usedPercent': 10}}}, - 3: {'summary': {}, 'dailyUsageBuckets': []}, -} -for response_id, result in responses.items(): - print(json.dumps({'id': response_id, 'result': result}), flush=True) -""" - ) - mock.chmod(0o755) - monkeypatch.setattr(mod, "CODEX_BIN", str(mock)) - monkeypatch.setenv("REQUEST_LOG", str(request_log)) - - rate, usage = mod.query_codex(timeout=5) - requests = [json.loads(line) for line in request_log.read_text().splitlines()] - - assert rate["rateLimits"]["primary"]["usedPercent"] == 10 - assert usage["summary"] == {} - assert [request["method"] for request in requests] == [ - "initialize", - "initialized", - "account/rateLimits/read", - "account/usage/read", - ] - assert all("/status" not in json.dumps(request) for request in requests) - - -def test_codex_cleanup_is_bounded_when_process_cannot_be_reaped(monkeypatch): - mod = load_module() - processes = [] - selectors = [] - - class StubbornProcess: - def __init__(self, *_args, **_kwargs): - self.stdin = StringIO() - self.stdout = StringIO( - "\n".join( - json.dumps({"id": response_id, "result": result}) - for response_id, result in ( - (1, {}), - (2, {"rateLimits": {}}), - (3, {"summary": {}, "dailyUsageBuckets": []}), - ) - ) - + "\n" - ) - self.terminate_calls = 0 - self.kill_calls = 0 - self.wait_timeouts = [] - processes.append(self) - - def terminate(self): - self.terminate_calls += 1 - - def kill(self): - self.kill_calls += 1 - - def wait(self, timeout): - self.wait_timeouts.append(timeout) - raise subprocess.TimeoutExpired("codex-mock", timeout) - - class TrackingSelector: - def __init__(self): - self.fileobj = None - self.closed = False - selectors.append(self) - - def register(self, fileobj, _events): - self.fileobj = fileobj - - def select(self, _timeout): - return [(SimpleNamespace(fileobj=self.fileobj), None)] - - def close(self): - self.closed = True - - monkeypatch.setattr(mod.subprocess, "Popen", StubbornProcess) - monkeypatch.setattr(mod.selectors, "DefaultSelector", TrackingSelector) - - try: - mod.query_codex(timeout=1) - except Exception as error: # The cleanup result must be sanitized and non-TimeoutExpired. - assert type(error) is RuntimeError - assert str(error) == "Codex app-server cleanup failed" - else: - raise AssertionError("unreaped Codex process was reported as successful") - - process = processes[0] - assert process.terminate_calls == 1 - assert process.kill_calls == 1 - assert process.wait_timeouts == [5, 5] - assert process.stdin.closed - assert process.stdout.closed - assert selectors[0].closed - - -def test_unexpected_provider_failure_is_sanitized_and_preserves_last_good(monkeypatch, capsys): +def test_unexpected_provider_failure_is_sanitized_and_preserves_last_good( + monkeypatch, capsys +): mod = load_module() collector = mod.Collector() previous = mod.Sample( diff --git a/testing/tests/test_hermes_ai_usage_exporter_coverage.py b/testing/tests/test_hermes_ai_usage_exporter_coverage.py new file mode 100644 index 00000000..affaf336 --- /dev/null +++ b/testing/tests/test_hermes_ai_usage_exporter_coverage.py @@ -0,0 +1,275 @@ +"""Focused edge coverage for AI usage exporter provider and polling boundaries.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from io import BytesIO +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" + + +def load_module(): + """Load a fresh exporter while making its sibling modules importable.""" + script_directory = str(SCRIPT.parent) + if script_directory not in sys.path: + sys.path.insert(0, script_directory) + name = "ai_usage_exporter_coverage" + spec = importlib.util.spec_from_file_location(name, SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_parser_rejects_nonfinite_and_malformed_optional_fields(): + mod = load_module() + + assert mod._number(True) is None + assert mod._number("1") is None + assert mod._number(float("nan")) is None + assert mod._number(float("inf")) is None + assert mod._timestamp(None) is None + assert mod._timestamp("not-a-time") is None + assert mod._window_name({}, "fallback") == "fallback" + assert mod._codex_limit_name("other", {}) == "additional" + + samples = mod.parse_codex_payloads( + { + "rateLimitsByLimitId": { + 1: {}, + "not-a-snapshot": "invalid", + "unnamed": { + "primary": {"usedPercent": None}, + "secondary": { + "usedPercent": 120, + "resetsAt": "invalid", + }, + }, + } + }, + { + "summary": { + "lifetimeTokens": None, + "currentStreakDays": 2, + "longestStreakDays": 3, + "longestRunningTurnSec": 4, + }, + "dailyUsageBuckets": [ + None, + {"startDate": "invalid", "tokens": 2}, + {"startDate": "2026-08-17", "tokens": None}, + ], + }, + ) + + assert any(sample.value == 120 for sample in samples) + assert not any( + sample.name == "atlas_ai_quota_reset_timestamp_seconds" for sample in samples + ) + assert mod.parse_codex_payloads({"rateLimits": None}, {"summary": None}) + + +def test_claude_parser_ignores_unbounded_fields_and_invalid_extra_usage(): + mod = load_module() + + assert mod.parse_claude_payload( + { + "five_hour": {"utilization": None}, + "seven_day": {"utilization": 10, "resets_at": None}, + "extra_usage": {"is_enabled": "yes"}, + } + ) + assert mod.parse_claude_payload({"extra_usage": "invalid"}) == [] + + +def test_provider_queries_validate_credentials_and_response_shape( + tmp_path, monkeypatch +): + mod = load_module() + credentials = tmp_path / "credentials.json" + credentials.write_text("{}", encoding="utf-8") + monkeypatch.setattr(mod, "CLAUDE_CREDENTIALS", credentials) + + with pytest.raises(RuntimeError, match="credentials are incomplete"): + mod.query_claude() + + credentials.write_text( + json.dumps({"claudeAiOauth": {"accessToken": "runtime-only-token"}}), + encoding="utf-8", + ) + bodies = iter([{"five_hour": {}}, []]) + requests = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self, maximum): + assert maximum == 1 << 20 + return json.dumps(next(bodies)).encode() + + def open_request(request, timeout): + requests.append(request) + assert timeout == 15 + return Response() + + monkeypatch.setattr(mod, "urlopen", open_request) + assert mod.query_claude() == {"five_hour": {}} + assert requests[0].get_header("Authorization") == "Bearer runtime-only-token" + with pytest.raises(RuntimeError, match="response is not an object"): + mod.query_claude() + + assert mod.query_codex is mod.codex_query.query_codex + + +def test_collector_refreshes_both_providers_and_contains_unknown_provider( + monkeypatch, capsys +): + mod = load_module() + collector = mod.Collector() + monkeypatch.setattr( + mod, + "query_codex", + lambda: ({"rateLimits": {"primary": {"usedPercent": 1}}}, {}), + ) + monkeypatch.setattr(mod, "query_claude", lambda: {"five_hour": {"utilization": 2}}) + + collector.refresh_provider("openai") + collector.refresh_provider("anthropic") + assert collector._providers["openai"].fetch_success + assert collector._providers["anthropic"].fetch_success + assert collector._providers["openai"].last_success > 0 + + collector.record_failure("openai") + assert not collector._providers["openai"].fetch_success + with pytest.raises(KeyError, match="unknown"): + collector.refresh_provider("unknown") + assert "unknown quota collection deferred: ValueError" in capsys.readouterr().out + + +def test_polling_failure_accounting_and_cycle_errors_are_sanitized(monkeypatch, capsys): + mod = load_module() + secret = "must-not-leak" + + class Failure(Exception): + pass + + class Collector: + def refresh_provider(self, _provider): + raise Failure(secret) + + def record_failure(self, *_args, **_kwargs): + raise Failure(secret) + + engine = mod.PollingEngine(Collector(), interval=60) + engine.poll_once() + output = capsys.readouterr().out + assert "quota failure accounting deferred: Failure" in output + assert secret not in output + + monkeypatch.setattr( + engine, "poll_once", lambda: (_ for _ in ()).throw(Failure(secret)) + ) + monkeypatch.setattr( + mod.time, "sleep", lambda _interval: (_ for _ in ()).throw(StopIteration) + ) + with pytest.raises(StopIteration): + engine.run() + output = capsys.readouterr().out + assert "quota polling cycle deferred: Failure" in output + assert secret not in output + + +def test_poller_start_is_idempotent_and_main_wires_server(monkeypatch): + mod = load_module() + threads = [] + + class Thread: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.started = False + threads.append(self) + + def start(self): + self.started = True + + monkeypatch.setattr(mod.threading, "Thread", Thread) + engine = mod.PollingEngine(mod.Collector(), interval=60) + engine.start() + engine.start() + assert len(threads) == 1 + assert threads[0].started + assert threads[0].kwargs["daemon"] is True + + events = [] + + class Poller: + def __init__(self, collector, *, interval): + events.append(("poller", collector, interval)) + + def start(self): + events.append("started") + + class Server: + def __init__(self, address, handler): + events.append(("server", address, handler)) + + def serve_forever(self): + events.append("served") + + monkeypatch.setattr(mod, "PollingEngine", Poller) + monkeypatch.setattr(mod, "Server", Server) + monkeypatch.setattr( + mod, "make_handler", lambda collector, poller: (collector, poller) + ) + monkeypatch.setenv("ATLAS_AI_USAGE_INTERVAL_SECONDS", "1") + monkeypatch.setenv("ATLAS_AI_USAGE_PORT", "19010") + + assert mod.main() == 0 + assert events[0][0] == "poller" + assert events[0][2] == 60 + assert events[2][1][1] == 19010 + assert events[-1] == "served" + + +def test_http_handler_covers_not_found_logging_and_connection_reset(): + mod = load_module() + + class Collector: + def render(self): + return b"metrics\n" + + class Poller: + def is_healthy(self): + return True + + handler_class = mod.make_handler(Collector(), Poller()) + handler = object.__new__(handler_class) + handler.path = "/missing" + statuses = [] + handler.send_error = statuses.append + handler.do_GET() + assert statuses == [404] + assert handler.log_message("ignored") is None + + class Reset(BytesIO): + def write(self, _body): + raise ConnectionResetError + + handler.path = "/metrics" + handler.send_response = lambda _status: None + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = Reset() + handler.do_GET() diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..601f39df 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -59,9 +59,7 @@ def test_auto_boundary_selects_only_a_public_switchyard_route(tmp_path, monkeypa assert source == "auto" -def test_legacy_maximum_default_migrates_to_adaptive_agent_route( - tmp_path, monkeypatch -): +def test_legacy_maximum_default_migrates_to_adaptive_agent_route(tmp_path, monkeypatch): path = tmp_path / "route-policy.json" path.write_text( json.dumps({"mode": "auto", "auto_route": "atlas/auto/maximum"}), @@ -90,9 +88,7 @@ def test_ui_priority_changes_auto_posture_without_selecting_a_target( assert route in router.AUTO_ROUTES -def test_ui_manual_model_and_effort_are_forwarded_as_constraints( - tmp_path, monkeypatch -): +def test_ui_manual_model_and_effort_are_forwarded_as_constraints(tmp_path, monkeypatch): monkeypatch.setattr(router, "POLICY_PATH", tmp_path / "route-policy.json") agent = _agent( model="atlas/manual/claude/opus", @@ -107,9 +103,7 @@ def test_ui_manual_model_and_effort_are_forwarded_as_constraints( ) -def test_cli_exact_manual_route_remains_exact_across_boundaries( - tmp_path, monkeypatch -): +def test_cli_exact_manual_route_remains_exact_across_boundaries(tmp_path, monkeypatch): monkeypatch.setattr(router, "POLICY_PATH", tmp_path / "route-policy.json") agent = _agent( model="atlas/manual/claude/opus/xhigh", @@ -222,39 +216,58 @@ def test_adapter_contains_no_content_classifier_or_direct_ollama_call(): def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch): status = sys.modules["hermes_auto_router"].provider_status_text.__globals__ module = sys.modules[status["provider_status_payload"].__module__] - monkeypatch.setattr(module, "_get_json", lambda url, timeout=3.0: ( - {"status": "ok"} if url.endswith("/health") else { - "total_requests": 3, - "total_errors": 1, - "total_tokens": {"total": 900}, - "models": { - "route/codex/sol/xhigh": { - "calls": 1, - "errors": 0, - "total_tokens": 600, - "avg_latency_ms": 1200, + monkeypatch.setattr( + module, + "_get_json", + lambda url, timeout=3.0: ( + {"status": "ok"} + if url.endswith("/health") + else { + "total_requests": 3, + "total_errors": 1, + "total_tokens": {"total": 900}, + "models": { + "route/codex/sol/xhigh": { + "calls": 1, + "errors": 0, + "total_tokens": 600, + "avg_latency_ms": 1200, + }, + "route/claude/sonnet/high": { + "calls": 0, + "errors": 1, + "total_tokens": 0, + }, + "route/local/qwen2.5-14b/medium": { + "calls": 1, + "errors": 0, + "total_tokens": 300, + }, }, - "route/claude/sonnet/high": { - "calls": 0, - "errors": 1, - "total_tokens": 0, - }, - "route/local/qwen2.5-14b/medium": { - "calls": 1, - "errors": 0, - "total_tokens": 300, - }, - }, - "routing_fallbacks": {"unavailable": 1}, - "classifier": {"total_requests": 3, "total_errors": 0}, - } - )) - monkeypatch.setattr(module, "_codex_account", lambda: { - "authenticated": True, "plan": "plus", "quota_reported": False, - }) - monkeypatch.setattr(module, "_claude_account", lambda: { - "authenticated": True, "plan": "max", "quota_reported": False, - }) + "routing_fallbacks": {"unavailable": 1}, + "classifier": {"total_requests": 3, "total_errors": 0}, + } + ), + ) + monkeypatch.setattr( + module, + "_codex_account", + lambda: { + "authenticated": True, + "plan": "plus", + "quota_reported": False, + }, + ) + monkeypatch.setattr( + module, + "_claude_account", + lambda: { + "authenticated": True, + "plan": "max", + "quota_reported": False, + }, + ) + monkeypatch.setattr(module, "_fresh_health", lambda *_args, **_kwargs: {}) payload = module.provider_status_payload() @@ -359,15 +372,20 @@ def test_claude_account_treats_a_live_refresh_token_as_refreshable( status = sys.modules["hermes_auto_router"].provider_status_text.__globals__ module = sys.modules[status["provider_status_payload"].__module__] path = tmp_path / ".credentials.json" - path.write_text(json.dumps({ - "claudeAiOauth": { - "accessToken": "expired-access", - "refreshToken": "live-refresh", - "expiresAt": 1, - "refreshTokenExpiresAt": 32_472_192_000, - "subscriptionType": "max", - } - }), encoding="utf-8") + path.write_text( + json.dumps( + { + "claudeAiOauth": { + "accessToken": "expired-access", + "refreshToken": "live-refresh", + "expiresAt": 1, + "refreshTokenExpiresAt": 32_472_192_000, + "subscriptionType": "max", + } + } + ), + encoding="utf-8", + ) monkeypatch.setattr(module, "CLAUDE_AUTH_PATH", path) account = module._claude_account() @@ -382,13 +400,18 @@ def test_codex_account_treats_a_refresh_token_as_refreshable(tmp_path, monkeypat status = sys.modules["hermes_auto_router"].provider_status_text.__globals__ module = sys.modules[status["provider_status_payload"].__module__] path = tmp_path / "auth.json" - path.write_text(json.dumps({ - "auth_mode": "chatgpt", - "tokens": { - "access_token": "expired-access", - "refresh_token": "live-refresh", - }, - }), encoding="utf-8") + path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": "expired-access", + "refresh_token": "live-refresh", + }, + } + ), + encoding="utf-8", + ) monkeypatch.setattr(module, "CODEX_AUTH_PATH", path) account = module._codex_account() @@ -406,7 +429,8 @@ def test_agent_mounts_provider_status_dashboard_into_auto_router_plugin(): (root / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") ) volume = next( - item for item in deployment["spec"]["template"]["spec"]["volumes"] + item + for item in deployment["spec"]["template"]["spec"]["volumes"] if item["name"] == "auto-router-plugin" ) paths = {item["path"] for item in volume["configMap"]["items"]} From d208a89d9d590bffb8269e6bfc27b45ceced0374 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:24:37 +0000 Subject: [PATCH 18/46] fix(hermes): close the zero-evidence fail-open in absence checks evaluate_names_absent returned PASS when its step exited 0 with no output, so five mandatory checks - the ones asserting that provider API keys, forge credentials, a cluster-admin binding, and shared coordinator state are absent - could report a pass on no evidence and turn a NO_GO into a GO. Both name rules now resolve their step through one guard in _line_step, so zero observations are NOT_RUN. Regressions pin all five real catalog specs plus both reachable silence paths: a POSIX pipeline whose status comes from its last stage, and a drifted kubectl -o jsonpath. The pool claim projection emits one = line per template volume so a volume without a PVC still counts as an observation rather than reading as drift. Also closes the review's reachable hardening and evidence defects: - pin Gitea paths to atlas/titan-iac on an exact segment boundary and reject relative segments, including percent-encoded ones - forbid impersonation structurally in every mode and vantage; the inner command of kubectl exec is re-checked rather than exempted, and validate_catalog no longer guards only the operator vantage - drop flux and helm from the binary allowlist; they had no pinned release digest, so no allowlisted binary can now be admitted that the executor would refuse to attest - remove the inert --concurrency and --expect-telegram-sessions flags and the dead concurrency bound; Telegram continuity stays mandatory - read the ephemeral pull index page by page, treat the create response as an authoritative source for the pull number, close every number either source names, and surface residue_ref plus exact manual_cleanup commands when creation is uncertain - keep executable_path and executable_sha256 on unrecorded bulk-evidence steps so withholding bytes never withholds binary attestation - revert the repo-wide hygiene legacy-exception mechanism; the contract change here is purely additive and the four pre-existing over-cap files are left to the canonical contract change in PR #14/#15 - correct the runbook ruff format scope so the documented command passes Split hermes_handoff_arming.py out of hermes_handoff_ephemeral.py to keep both modules under the 500-line cap. All 16 handoff modules hold at least 95% line and branch coverage; the mutation gate is 13/13. Co-Authored-By: Claude Opus 5 --- docs/hermes_full_handoff_acceptance.md | 97 ++++- scripts/ops/hermes_handoff_acceptance.py | 11 - scripts/ops/hermes_handoff_arming.py | 154 ++++++++ scripts/ops/hermes_handoff_catalog.py | 1 - scripts/ops/hermes_handoff_checks_workers.py | 9 +- scripts/ops/hermes_handoff_ephemeral.py | 358 ++++++++---------- scripts/ops/hermes_handoff_evaluators.py | 20 +- scripts/ops/hermes_handoff_exec.py | 19 +- scripts/ops/hermes_handoff_policy.py | 85 ++--- scripts/ops/hermes_handoff_run.py | 9 +- testing/quality_contract.json | 25 +- testing/quality_handoff_mutation.py | 52 +++ testing/quality_hygiene.py | 25 +- .../tests/test_hermes_handoff_acceptance.py | 4 +- .../tests/test_hermes_handoff_ephemeral.py | 141 ++++++- testing/tests/test_hermes_handoff_exec.py | 3 - .../test_hermes_handoff_mutation_gate.py | 13 +- testing/tests/test_hermes_handoff_policy.py | 76 +++- testing/tests/test_hermes_handoff_run.py | 27 +- .../test_hermes_handoff_zero_evidence.py | 243 ++++++++++++ testing/tests/test_quality_contract.py | 33 +- 21 files changed, 977 insertions(+), 428 deletions(-) create mode 100644 scripts/ops/hermes_handoff_arming.py create mode 100644 testing/tests/test_hermes_handoff_zero_evidence.py diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index be13c1f0..5533ef2d 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -58,13 +58,24 @@ Default mode permits a narrow read grammar: projections, `auth reconcile`, config operations, all mutations, and every dry-run mutation trick are rejected before spawn. - Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, - external diff helpers, and push are rejected. Helm values are not readable. + external diff helpers, and push are rejected. +- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a + look-alike repository such as `atlas/titan-iac-evil` is refused, and any + relative segment — including a percent-encoded `%2e%2e` — is rejected before + the request is built. - Shell commands must be rendered from frozen templates, use fixed executable paths, and cannot name service-account, Vault, runtime-access, SSH, or other credential roots. -- Outer impersonation is rejected. A reviewed inner command can use impersonation - only after an explicit `kubectl exec --` boundary; it cannot substitute for the - independent self identity. +- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid` + are refused for every argv, in every mode, from every vantage, and the inner + command of a `kubectl exec` is re-checked under the same rule rather than being + granted an exemption. `validate_catalog` refuses an impersonating step from any + vantage, not only the operator. A future audited self-probe that genuinely needs + impersonation is a reviewed code change, not a configuration flag. +- The binary allowlist is exactly the set with pinned release digests: + `kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are + not on it; Flux and Helm status is read through `kubectl` instead, so no + allowlisted binary can be admitted that the executor would refuse to attest. - `PATH` is replaced with a fixed path. Every outer executable is resolved to an allowed real path, checked for unsafe write mode, and compared with a pinned SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate @@ -73,9 +84,14 @@ Default mode permits a narrow read grammar: The executor uses a new process group, drains stdout and stderr concurrently, and retains at most the configured bytes while the child is running. A monotonic absolute timeout terminates and reaps the group, including descendants that keep -pipes open. Timeout, deadline, output, and concurrency values reject booleans, -zero/negative values, non-finite numbers, and values outside fixed bounds. +pipes open. Timeout, deadline, and output-byte values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. There +is no concurrency knob: `run_catalog` is strictly sequential and says so rather +than accepting a bound it would ignore. Truncation discards the affected stream and makes every evaluator `NOT_RUN`. +Bulk-evidence steps have their bytes replaced before they are recorded, but they +keep the resolved executable path and its attested SHA-256, so withholding output +never withholds the provenance of the binary that produced it. Credential screening runs before truncation. Command, output, error, identity, reason, evidence, JSON, and human-summary fields are screened recursively, @@ -99,6 +115,17 @@ forge administrative denial check performs a read-only GET and accepts only an explicit authorization refusal; generic `404`, `not found`, and `no route` text are not denial evidence. +An absence is only evidence when the probe observed something. Both name rules +resolve their step through one guard, so a step that exits `0` with no lines — or +with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice +two ways, and both are pinned by regressions: a POSIX pipeline reports the status +of its *last* stage, so a missing upstream binary in the frozen `env_names` +template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the +structure it walks has drifted, whether a parent field, a leaf field, or a +container filter. Because all five absence checks are mandatory, silence costs +the release its `GO`. The pool claim projection emits one `=` line +per template volume so a volume without a PVC still counts as an observation. + The strict evaluators reject duplicate-key JSON, raw control characters, `NaN`/infinity, boolean/number equality coercion, future timestamps, stale provider evidence, and every malformed routing or Flux row. Routing freshness is @@ -143,11 +170,22 @@ then pinned to that exact ref. It allows only its push, one exact draft-create payload, cleanup of the exactly discovered PR number, and deletion of that ref. The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the -pushed SHA, always discovers the PR after create (including ambiguous create -responses), and requires exactly one open draft with the expected base, head ref, -base SHA, and head SHA. Cleanup requires successful PR close, successful branch deletion, -a successful empty `ls-remote`, and a strict re-read of the same closed PR. -Malformed or partial cleanup is `FAIL`; it is never inferred from an error. +pushed SHA, then identifies the pull request from two independent sources: the +create response itself, and the pull index read page by page to its last page +(50 per page, 20 pages maximum) rather than a single truncated first page. Either +source alone names a pull request this run definitely made, because preflight +proved the ephemeral ref did not exist. Every number found from either source is +registered and closed, so a failed or unusable index can no longer leave an +orphan open. A `GO` still requires exactly one open draft with the expected base, +head ref, base SHA, and head SHA, and the two sources must agree. + +Cleanup requires successful PR close, successful branch deletion, a successful +empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or +partial cleanup is `FAIL`; it is never inferred from an error. When anything is +uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact +`manual_cleanup` command list — the `PATCH .../pulls/ --field state=closed` +calls for every number found, the `DELETE .../branches/` call, and, when no +number could be determined at all, the exact ref to sweep the index for by hand. Force, force-with-lease, fan-out, protected refs, other repositories, other PR numbers, arbitrary PATCH payloads, and repository administration remain denied. @@ -224,28 +262,51 @@ rollback. ## Quality and release evidence -The canonical quality contract explicitly manages, lints, and tracks all 15 +The canonical quality contract explicitly manages, lints, and tracks all 16 `scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and branch coverage per tracked file and the 500-physical-line source cap. Contract tests derive the module set so a future file cannot silently escape the gate. -Four pre-existing over-cap test files are temporarily locked to their exact -line count and SHA-256; any byte edit invalidates the exception and must split -the file below 500 lines. PR #14 supplies the intended additive test splits. +This change is purely additive to the contract: it registers its own modules and +changes no repo-wide hygiene semantics. + +The gate's `hygiene` step is red on `main` and stays red here. Exactly four +pre-existing test files exceed the 500-line cap — +`scripts/tests/test_dashboards_render_atlas.py` (516), +`testing/tests/test_hermes_chat_quality.py` (2242), +`testing/tests/test_hermes_cli_lanes.py` (1971), and +`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at +merge-base `30259b52` and at current `origin/main`, none is touched by this +change, and the same four are the only issues a pristine `main` reports. Splitting +them, or grandfathering them, is a repo-wide contract decision that belongs to the +canonical contract change carried by PR #14/#15, not to this acceptance harness. +Every other gate step — docs, smell, unit, coverage — is green, and every file +this change adds or edits is under the cap. ```sh python3 -m testing.quality_gate --profile local --build-dir build python3 -m testing.quality_handoff_mutation python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ - scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py + scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \ + testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py python3 -m ruff format --check \ - scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py + scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \ + testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py python3 -m compileall -q scripts/ops testing kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - git diff --check origin/main...HEAD ``` +The format scope names the files this change owns. A wider `testing/quality_*.py` +glob picks up `testing/quality_gate.py`, which is unmodified here and not +format-clean, so the documented command would fail for a reason this change did +not cause. `ruff format` is evidence, not a gate step: the canonical gate runs +`ruff check` only. + The deterministic mutation gate must kill every policy, redaction, verdict, -numeric, truncation, and cleanup mutant. The adversarial suite includes large +numeric, truncation, provenance, and cleanup mutant, including one that restores +the zero-evidence pass, one that restores the bare-prefix repository pin, one +that stops re-checking a `kubectl exec` inner command, and one that drops binary +attestation from an unrecorded step. The adversarial suite includes large concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py index 6b35d49f..6dbc69d2 100755 --- a/scripts/ops/hermes_handoff_acceptance.py +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -123,9 +123,6 @@ def build_parser() -> argparse.ArgumentParser: default=DEFAULT_MAX_BYTES, help="Per-capture byte budget.", ) - parser.add_argument( - "--concurrency", type=int, default=1, help="Validated scheduling bound." - ) parser.add_argument( "--baseline-commit", default=Targets.baseline_commit, @@ -190,12 +187,6 @@ def build_parser() -> argparse.ArgumentParser: default=0, help="Chat tenant ordinal that carries the Telegram identity.", ) - parser.add_argument( - "--expect-telegram-sessions", - action="store_true", - default=True, - help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", - ) parser.add_argument( "--pool-worker-env", action="append", @@ -243,7 +234,6 @@ def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Target pool_replicas=arguments.pool_replicas, chat_config_revision=arguments.chat_config_revision, chat_ordinal=arguments.chat_ordinal, - expect_telegram_sessions=arguments.expect_telegram_sessions, pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), max_evidence_age_seconds=arguments.max_evidence_age, now=now, @@ -329,7 +319,6 @@ def main(argv: Sequence[str] | None = None) -> int: command_timeout=arguments.command_timeout, deadline_seconds=arguments.deadline, max_bytes=arguments.max_output_bytes, - concurrency=arguments.concurrency, ) except ValueError as exc: report = unavailable_report( diff --git a/scripts/ops/hermes_handoff_arming.py b/scripts/ops/hermes_handoff_arming.py new file mode 100644 index 00000000..5aa596c4 --- /dev/null +++ b/scripts/ops/hermes_handoff_arming.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Arming, provenance, and push-target rules for the armed ephemeral mode. + +Nothing here touches the network or spawns a process. It is the half of armed +mode that must be true before a single command exists: the confirmation phrase, +the fixed repository and base, the one ephemeral ref grammar, and a local +worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release. +""" + +from __future__ import annotations + +import configparser +import re +from dataclasses import dataclass +from pathlib import Path + +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + arm_ephemeral_policy, +) + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Every fixed and caller-confirmed input to one armed run.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" + + @property + def ref(self) -> str: + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + return name.strip().removeprefix("refs/heads/").strip("/").lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is the one ephemeral branch grammar.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") + assert_push_target_allowed(request.ref) + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py index 9dc5c98a..59334dec 100644 --- a/scripts/ops/hermes_handoff_catalog.py +++ b/scripts/ops/hermes_handoff_catalog.py @@ -152,7 +152,6 @@ class Targets: routing_tail_bytes: int = 512 * 1024 listing_bytes: int = 512 * 1024 git_askpass: str = "/opt/coordinator/gitea_askpass.sh" - expect_telegram_sessions: bool = True now: Any = None extra: dict[str, Any] = field(default_factory=dict) diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py index eba7df6d..d62c719c 100644 --- a/scripts/ops/hermes_handoff_checks_workers.py +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -102,12 +102,15 @@ def _pool(targets: Targets) -> list[CheckSpec]: f"statefulset/{targets.pool_statefulset}", "-o", projection( - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + "jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}" + "{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace( + "'", '"' + ) ), ) ], - {"step": "claims", "names": targets.coordinator_claims}, - rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", + {"step": "claims", "contains": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.", ), *_pool_env(targets), ] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py index fdcfdbad..ca1ea594 100644 --- a/scripts/ops/hermes_handoff_ephemeral.py +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -1,161 +1,57 @@ #!/usr/bin/env python3 -"""Strictly preflighted, discoverable, and self-cleaning armed mode.""" +"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run. + +The create response — not the index — is the authoritative name of a pull +request this harness made, and the index is read to its last page. When either +source is uncertain the run fails closed and reports the exact residue plus the +exact commands that remove it, so an operator is never left guessing what was +created. + +Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming` +and are re-exported here for callers that treat armed mode as one surface. +""" from __future__ import annotations -import configparser -import re from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path from typing import Any +from hermes_handoff_arming import ( + CONFIRMATION, + REF_PREFIX, + ArmingError, + ArmRequest, + assert_push_target_allowed, + bounded_read, + normalise_branch, + preflight, +) from hermes_handoff_exec import Outcome, Runner, Vantage from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec from hermes_handoff_policy import ( EXPECTED_BASE, - EXPECTED_REMOTE, - EXPECTED_REPO, GITEA_CLIENT, - arm_ephemeral_policy, register_ephemeral_pull, ) from hermes_handoff_rules import strict_json -CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" -REF_PREFIX = "ephemeral/hermes-handoff-acceptance" -REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") -TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") -SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +__all__ = [ + "CONFIRMATION", + "DISCOVERY_PAGE_BUDGET", + "DISCOVERY_PAGE_SIZE", + "REF_PREFIX", + "ArmRequest", + "ArmingError", + "assert_push_target_allowed", + "bounded_read", + "normalise_branch", + "preflight", + "run_armed", +] + DRAFT_TITLE_PREFIX = "WIP: " -EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" -EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" -PROTECTED_NAMES = frozenset( - { - "default", - "develop", - "head", - "main", - "master", - "prod", - "production", - "release", - "stable", - "trunk", - } -) - - -class ArmingError(ValueError): - """Raised when an ephemeral run is not safe to start.""" - - -@dataclass(frozen=True) -class ArmRequest: - """Every fixed and caller-confirmed input to one armed run.""" - - repo: str - remote: str - token: str - confirmation: str - base: str = EXPECTED_BASE - expected_head: str = "" - expected_base_sha: str = "" - - @property - def ref(self) -> str: - return f"{REF_PREFIX}/{self.token}" - - -def normalise_branch(name: str) -> str: - """Return a branch name stripped of ref decoration, for comparison.""" - return name.strip().removeprefix("refs/heads/").strip("/").lower() - - -def assert_push_target_allowed(ref: str) -> None: - """Raise unless ``ref`` is the one ephemeral branch grammar.""" - if normalise_branch(ref) in PROTECTED_NAMES: - raise ArmingError(f"refusing to push to protected branch {ref!r}") - if not REF_RE.fullmatch(ref): - raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") - - -def _bounded_read(path: Path, limit: int = 1024 * 1024) -> str: - data = path.read_bytes() - if not data or len(data) > limit: - raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") - return data.decode("utf-8", "strict") - - -def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: - """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" - resolved = root.resolve(strict=True) - marker = resolved / ".git" - if not marker.is_file(): - raise ArmingError( - "armed mode must run at the root of the existing linked worktree" - ) - line = _bounded_read(marker, 4096).strip() - if not line.startswith("gitdir: "): - raise ArmingError("linked-worktree Git metadata is malformed") - gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) - common = (gitdir / _bounded_read(gitdir / "commondir", 4096).strip()).resolve( - strict=True - ) - head = _bounded_read(gitdir / "HEAD", 4096).strip() - expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" - if head != f"ref: {expected_ref}": - raise ArmingError("armed mode must run from the existing PR #19 feature branch") - ref_path = common / expected_ref - actual_head = _bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" - if actual_head != expected_head: - raise ArmingError( - "local worktree HEAD does not match the exact reviewed PR head" - ) - remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE - remote_main = ( - _bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" - ) - if remote_main != expected_base_sha: - raise ArmingError("local origin/main does not match the exact release base SHA") - parser = configparser.RawConfigParser(interpolation=None) - try: - parser.read_string(_bounded_read(common / "config")) - remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") - except (configparser.Error, KeyError) as exc: - raise ArmingError("local Git remote metadata is malformed") from exc - if remote_url != EXPECTED_REMOTE_URL: - raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") - - -def preflight( - request: ArmRequest, expected_repo: str, worktree: Path | None = None -) -> None: - """Validate arming and local provenance before any subprocess/network call.""" - if request.confirmation != CONFIRMATION: - raise ArmingError("confirmation phrase does not match") - if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: - raise ArmingError( - "armed repository is not the fixed Atlas titan-iac repository" - ) - if ( - request.remote != EXPECTED_REMOTE - or normalise_branch(request.base) != EXPECTED_BASE - ): - raise ArmingError("armed remote and pull-request base are fixed to origin/main") - if not TOKEN_RE.fullmatch(request.token): - raise ArmingError( - "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" - ) - if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( - request.expected_base_sha - ): - raise ArmingError("exact reviewed head and release base SHAs are required") - assert_push_target_allowed(request.ref) - _attest_worktree( - worktree or Path.cwd(), request.expected_head, request.expected_base_sha - ) - arm_ephemeral_policy(request.ref) +DISCOVERY_PAGE_SIZE = 50 +DISCOVERY_PAGE_BUDGET = 20 def _spec(identifier: str, title: str) -> CheckSpec: @@ -246,17 +142,74 @@ def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: return problems -def _discover(outcome: Outcome, request: ArmRequest) -> tuple[list[int], str]: +def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]: + """Return the number the create response itself reports. + + The create response is the only source that names a pull request the harness + definitely made. Relying on the index alone means an index that fails or + answers ambiguously leaves an unnamed, unclosed pull request behind. + """ if not outcome.ok: - return ([], f"pull-request discovery failed: {_detail(outcome)}") + return (0, f"pull-request creation failed: {_detail(outcome)}") try: payload = strict_json(outcome.stdout) except ValueError as exc: - return ([], f"pull-request discovery returned malformed JSON: {exc}") - if not isinstance(payload, list): - return ([], "pull-request discovery did not return an array") + return (0, f"pull-request creation returned malformed JSON: {exc}") + problems = _pr_problems(payload, request) + if problems: + return (0, f"created pull request is invalid: {', '.join(problems)}") + return (payload["number"], "") + + +def _index_pages( + runner: Runner, vantage: Vantage, api: str, reserve: float +) -> tuple[list[Outcome], list[Any], str]: + """Read every page of the pull index; a truncated page can hide a match.""" + outcomes: list[Outcome] = [] + rows: list[Any] = [] + for page in range(1, DISCOVERY_PAGE_BUDGET + 1): + outcome = runner.run( + ( + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}", + ), + vantage, + reserve_seconds=reserve, + ) + outcomes.append(outcome) + if not outcome.ok: + return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}") + if not isinstance(payload, list): + return (outcomes, [], f"discovery page {page} did not return an array") + rows.extend(payload) + if len(payload) < DISCOVERY_PAGE_SIZE: + return (outcomes, rows, "") + return (outcomes, [], "pull-request discovery exceeded its page budget") + + +def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]: + """Name the exact residue and the exact commands that remove it by hand.""" + manual = [ + f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed" + for number in numbers + ] + if not numbers: + manual.append( + f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}" + f" then close by hand every pull whose head ref is {request.ref}" + ) + manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}") + return {"residue_ref": request.ref, "manual_cleanup": manual} + + +def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]: matches: list[int] = [] - for index, item in enumerate(payload): + for index, item in enumerate(rows): if ( not isinstance(item, dict) or not isinstance(item.get("head"), dict) @@ -354,33 +307,34 @@ def _cleanup( ) +def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]: + """Report the two results that follow a run which created no artifact.""" + return [ + branch, + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + draft_reason, + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + + def run_armed( runner: Runner, vantage: Vantage, request: ArmRequest ) -> list[CheckResult]: """Create, discover, validate, and clean one exact ephemeral branch/PR.""" guard = _protected_refusal_result() + branch_title = "A unique ephemeral feature branch can be pushed and removed" if guard.status != PASS: - return [ - guard, - _result( - "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", - NOT_RUN, - "protected-ref guard failed", - ), - _result( - "draft-pull-request", - "A draft pull request can be opened and closed", - NOT_RUN, - "protected-ref guard failed", - ), - _result( - "cleanup-verified", - "Every ephemeral artifact is removed", - NOT_RUN, - "nothing was created", - ), - ] + unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed") + return [guard, *_nothing_created(unrun, "protected-ref guard failed")] api = f"/api/v1/repos/{request.repo}" reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) existing = runner.run( @@ -392,28 +346,8 @@ def run_armed( reason = ( _detail(existing) if not existing.ok else "the ephemeral ref already exists" ) - return [ - guard, - _result( - "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", - FAIL, - reason, - [existing], - ), - _result( - "draft-pull-request", - "A draft pull request can be opened and closed", - NOT_RUN, - "branch preflight failed", - ), - _result( - "cleanup-verified", - "Every ephemeral artifact is removed", - NOT_RUN, - "nothing was created", - ), - ] + refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing]) + return [guard, *_nothing_created(refused, "branch preflight failed")] local_head = runner.run( ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve ) @@ -441,7 +375,7 @@ def run_armed( ] branch_result = _result( "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", + branch_title, PASS if branch_ok else FAIL, "the exact reviewed HEAD was pushed" if branch_ok @@ -450,7 +384,7 @@ def run_armed( {"ref": request.ref, "head": request.expected_head}, ) numbers: list[int] = [] - discovery_error = "" + problems: list[str] = ["branch creation was not verified"] create_outcomes: list[Outcome] = [] if branch_ok: created = runner.run( @@ -470,26 +404,36 @@ def run_armed( vantage, reserve_seconds=reserve, ) - discovered = runner.run( - (GITEA_CLIENT, "GET", f"{api}/pulls?state=all&limit=50"), - vantage, - reserve_seconds=reserve, + number, create_error = _created_number(created, request) + pages, rows, index_error = _index_pages(runner, vantage, api, reserve) + found, match_error = ( + _match_pulls(rows, request) if not index_error else ([], "") ) - create_outcomes = [created, discovered] - numbers, discovery_error = _discover(discovered, request) - for number in numbers: - register_ephemeral_pull(number) - else: - discovery_error = "branch creation was not verified" + create_outcomes = [created, *pages] + # Either source alone names a pull request this run definitely made: the + # ephemeral ref was proven absent in preflight, so any pull on it is ours. + numbers = sorted({item for item in (number, *found) if item}) + problems = [text for text in (index_error, match_error) if text] + if number and found and found != [number]: + problems.append(f"created #{number} is not the discovered {found}") + if not numbers: + problems.append( + create_error or "no exact draft pull request was identified" + ) + for item in numbers: + register_ephemeral_pull(item) + exact = len(numbers) == 1 and not problems draft_result = _result( "draft-pull-request", "A draft pull request can be opened and closed against the ephemeral branch", - PASS if len(numbers) == 1 and not discovery_error else FAIL, + PASS if exact else FAIL, f"discovered exact draft pull request #{numbers[0]}" - if len(numbers) == 1 and not discovery_error - else discovery_error, + if exact + else "; ".join(problems), create_outcomes, - {"pull_requests": numbers}, + {"pull_requests": numbers} + if exact + else {"pull_requests": numbers, **_residue(request, numbers, api)}, ) - cleanup = _cleanup(runner, vantage, request, api, numbers, discovery_error) + cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems)) return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py index 61e9e06b..ea0a21b8 100644 --- a/scripts/ops/hermes_handoff_evaluators.py +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -7,6 +7,9 @@ an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, not what the API server does — so a real request must have been made and refused before it will pass. +An absence is only evidence when the probe observed something, so a name step +that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass. + Importing this module also registers the JSON-shaped rules, so a caller only has to import one place to have the full catalog vocabulary available. """ @@ -129,16 +132,20 @@ def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: def _line_step( spec: CheckSpec, outcomes: dict[str, Outcome] ) -> tuple[set[str], str] | Evaluation: + """Return a step's distinct lines, or the NOT_RUN that silence must produce.""" key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") - return (set(lines(outcome)), key) + observed = set(lines(outcome)) + if not observed: + return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine") + return (observed, key) @evaluator("names_absent") def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: - """Pass when no forbidden name or substring appears in a step's lines.""" + """Pass when a step observed names and none of them is forbidden.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked @@ -179,8 +186,6 @@ def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eva return Evaluation( FAIL, f"required names absent: {', '.join(missing)}", evidence ) - if not observed: - return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) return Evaluation(PASS, "every required name is present", evidence) @@ -317,11 +322,8 @@ def evaluate_vantages_agree( @evaluator("not_armed") def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: """Report the fixed NOT_RUN a default, read-only run must produce.""" - return Evaluation( - NOT_RUN, - spec.expect.get("reason", "ephemeral mutation mode is not armed"), - {"armed": False}, - ) + reason = spec.expect.get("reason", "ephemeral mutation mode is not armed") + return Evaluation(NOT_RUN, reason, {"armed": False}) # This validator is one compact, linear state machine so every exit observes diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py index 08888b13..d3edbe09 100644 --- a/scripts/ops/hermes_handoff_exec.py +++ b/scripts/ops/hermes_handoff_exec.py @@ -37,7 +37,6 @@ DEFAULT_CLEANUP_RESERVE = 90.0 MAX_COMMAND_TIMEOUT = 600.0 MAX_RUN_DEADLINE = 7200.0 MAX_OUTPUT_BYTES = 8 * 1024 * 1024 -MAX_CONCURRENCY = 16 READ_CHUNK = 64 * 1024 PIPE_DRAIN_SECONDS = 0.25 @@ -52,8 +51,6 @@ FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", "LC_ALL": "C", "PATH": SAFE_PATH} EXPECTED_PATHS = { "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, - "flux": {"/opt/data/tools/bin/flux", "/usr/local/bin/flux", "/usr/bin/flux"}, - "helm": {"/opt/data/tools/bin/helm", "/usr/local/bin/helm", "/usr/bin/helm"}, "git": {"/usr/bin/git", "/bin/git"}, "hermes": {"/opt/hermes/.venv/bin/hermes"}, "sh": {"/usr/bin/sh", "/bin/sh"}, @@ -71,7 +68,7 @@ DEADLINE_ERROR = "deadline-exceeded" TIMEOUT_ERROR = "timeout" OPERATOR = "operator" SELF = "self" -CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") +CONTEXT_AWARE_BINARIES = ("kubectl",) POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} # fmt: on @@ -187,18 +184,6 @@ def _byte_budget(value: Any) -> int: return value -def _concurrency(value: Any) -> int: - if ( - isinstance(value, bool) - or not isinstance(value, int) - or not 1 <= value <= MAX_CONCURRENCY - ): - raise ValueError( - f"concurrency must be an integer between 1 and {MAX_CONCURRENCY}" - ) - return value - - def build_environment( vantage: Vantage, source: Mapping[str, str] | None = None ) -> dict[str, str]: @@ -408,7 +393,6 @@ class Runner: command_timeout: float = DEFAULT_COMMAND_TIMEOUT, deadline_seconds: float = DEFAULT_RUN_DEADLINE, max_bytes: int = DEFAULT_MAX_BYTES, - concurrency: int = 1, clock: Callable[[], float] = time.monotonic, executor: Callable[..., Capture] | None = None, spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, @@ -423,7 +407,6 @@ class Runner: "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE ) self.max_bytes = _byte_budget(max_bytes) - self.concurrency = _concurrency(concurrency) self._clock = clock self._executor = executor or ( _injected_executor(spawn) if spawn else execute_bounded diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py index fc1a7097..11c8bc1c 100644 --- a/scripts/ops/hermes_handoff_policy.py +++ b/scripts/ops/hermes_handoff_policy.py @@ -3,12 +3,19 @@ Before spawn it admits only credential-safe, structurally read-only grammar; shell is limited to frozen templates with absolute tool paths. + +Impersonation is structurally impossible: ``--as`` and its siblings are refused +for every argv, in every mode, from every vantage, and the inner command of a +``kubectl exec`` is re-checked under the same rule rather than being trusted. +Only binaries with a pinned release digest are on the allowlist, so nothing can +be admitted here that the executor would then refuse to attest. """ from __future__ import annotations import re from collections.abc import Iterator +from urllib.parse import unquote READ_ONLY = "read-only" ARMED = "ephemeral-armed" @@ -27,9 +34,7 @@ IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") FORBIDDEN_BINARIES = frozenset( {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} ) -ALLOWED_BINARIES = frozenset( - {"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT} -) +ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT}) TRUSTED_EXECUTABLE_ROOTS = ( "/bin/", "/usr/bin/", @@ -99,10 +104,6 @@ UNSAFE_PROJECTION_PARTS = ( SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip -FLUX_READ_SUBCOMMANDS = frozenset( - {"check", "events", "get", "stats", "trace", "tree", "version"} -) -HELM_READ_SUBCOMMANDS = frozenset({"history", "list", "status", "version"}) GIT_READ_SUBCOMMANDS = frozenset( { "cat-file", @@ -118,7 +119,6 @@ GIT_READ_SUBCOMMANDS = frozenset( "status", } ) -GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) GIT_FORCE_FLAGS = frozenset( {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} ) @@ -270,15 +270,16 @@ def _binary_name(value: str) -> str: return value.rsplit("/", 1)[-1] -def _reject_forbidden_arguments( - argv: tuple[str, ...], allow_impersonation: bool -) -> None: - permitted = IMPERSONATION_ARGS if allow_impersonation else () +def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None: + """Refuse credential and impersonation arguments up to the ``--`` boundary. + + Everything after ``--`` is the inner command of a ``kubectl exec``, which + :func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to + exactly this rule rather than being exempted from it. + """ boundary = argv.index("--") if "--" in argv else len(argv) for argument in argv[:boundary]: for forbidden in FORBIDDEN_ARGS: - if forbidden in permitted: - continue if argument == forbidden or argument.startswith(f"{forbidden}="): raise PolicyError(f"forbidden argument: {forbidden}") if argument == "--raw" or argument.startswith("--raw="): @@ -332,7 +333,7 @@ def _check_projection(output: str) -> None: ) -def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: +def _check_kubectl(argv: tuple[str, ...], mode: str) -> None: words = list(positionals(argv)) if not words or words[0] not in KUBECTL_SUBCOMMANDS: raise PolicyError("kubectl subcommand is outside the read-only boundary") @@ -351,7 +352,7 @@ def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) raise PolicyError( "kubectl exec inner command must use a fixed trusted path" ) - check_argv(inner, mode, allow_impersonation=True) + check_argv(inner, mode) return resources = words[1].split(",") if len(words) > 1 else [] if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): @@ -362,21 +363,6 @@ def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) _check_projection(output) -def _first_positional(binary: str, argv: tuple[str, ...]) -> str: - word = next(positionals(argv), None) - if word is None: - raise PolicyError(f"{binary} requires a subcommand") - return word - - -def _check_read_only_tool( - binary: str, argv: tuple[str, ...], allowed: frozenset[str] -) -> None: - subcommand = _first_positional(binary, argv) - if subcommand not in allowed: - raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") - - def _check_git(argv: tuple[str, ...], mode: str) -> None: unsafe = False for item in argv[1:]: @@ -390,7 +376,9 @@ def _check_git(argv: tuple[str, ...], mode: str) -> None: unsafe = True if unsafe: raise PolicyError("git configuration and helper overrides are forbidden") - subcommand = _first_positional("git", argv) + subcommand = next(positionals(argv), None) + if subcommand is None: + raise PolicyError("git requires a subcommand") if subcommand in GIT_READ_SUBCOMMANDS: return if subcommand == "push" and mode == ARMED: @@ -416,17 +404,34 @@ def _check_hermes(argv: tuple[str, ...]) -> None: ) +def _reject_dot_segments(path: str) -> None: + """Refuse any relative segment, including a percent-encoded one. + + The repository pin below is a textual prefix test, so a single ``..`` would + let a pinned path address another repository or the token endpoints once a + router normalised it. Decode first: ``%2e%2e`` is the same segment. + """ + decoded = unquote(unquote(path)).split("?", 1)[0] + if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]): + raise PolicyError("gitea_api.py paths may not contain relative segments") + + def _check_gitea(argv: tuple[str, ...], mode: str) -> None: if len(argv) < 3: raise PolicyError("gitea_api.py requires a method and a path") method, path = argv[1].upper(), argv[2] if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): raise PolicyError("gitea_api.py requires a bounded API path") - if path != "/api/v1/user" and not path.startswith(f"/api/v1/repos/{EXPECTED_REPO}"): + _reject_dot_segments(path) + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if ( + path != "/api/v1/user" + and path != repo_root + and not path.startswith((f"{repo_root}/", f"{repo_root}?")) + ): raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") if method in GITEA_READ_METHODS: return - repo_root = f"/api/v1/repos/{EXPECTED_REPO}" if mode == ARMED: fields = [ argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" @@ -463,9 +468,7 @@ def _check_shell(argv: tuple[str, ...]) -> None: raise PolicyError("sh is only reachable as sh -c ") -def check_argv( - argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False -) -> None: +def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None: """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" if mode not in MODES: raise PolicyError(f"unknown mode: {mode}") @@ -476,16 +479,12 @@ def check_argv( raise PolicyError(f"forbidden binary: {binary}") if binary not in ALLOWED_BINARIES: raise PolicyError(f"binary is not on the harness allowlist: {binary}") - _reject_forbidden_arguments(argv, allow_impersonation) + _reject_forbidden_arguments(argv) _reject_secret_paths(argv) if binary == "sh": _check_shell(argv) elif binary == "kubectl": - _check_kubectl(argv, mode, allow_impersonation) - elif binary == "flux": - _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) - elif binary == "helm": - _check_read_only_tool("helm", argv, HELM_READ_SUBCOMMANDS) + _check_kubectl(argv, mode) elif binary == "git": _check_git(argv, mode) elif binary == "hermes": diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py index a3b21cd7..45459dc8 100644 --- a/scripts/ops/hermes_handoff_run.py +++ b/scripts/ops/hermes_handoff_run.py @@ -97,9 +97,10 @@ def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: problems.append( f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" ) - if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): + if set(step.argv) & set(IMPERSONATION_ARGS): problems.append( - f"{spec.id}: step {step.key} impersonates from the operator vantage" + f"{spec.id}: step {step.key} impersonates from the " + f"{step.vantage} vantage" ) try: addressed = ( @@ -152,7 +153,7 @@ def validate_targets(targets: Targets) -> list[str]: problems.append("deployment revision must be a positive integer string") if targets.node_count != 3 or targets.pool_replicas != 3: problems.append("node and pool counts are fixed at exactly three") - if not targets.expect_telegram_sessions or not targets.chat_config_revision.strip(): + if not targets.chat_config_revision.strip(): problems.append( "Telegram continuity and its chat config revision are mandatory" ) @@ -386,6 +387,8 @@ def run_check( truncated=outcome.truncated, duration_ms=outcome.duration_ms, error=outcome.error, + executable_path=outcome.executable_path, + executable_sha256=outcome.executable_sha256, ) ) evaluation = evaluate(spec, outcomes) diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 9c3de171..a14e045d 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -15,6 +15,7 @@ ], "managed_modules": [ "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", "scripts/ops/hermes_handoff_catalog.py", "scripts/ops/hermes_handoff_checks_access.py", "scripts/ops/hermes_handoff_checks_delivery.py", @@ -52,6 +53,7 @@ ], "lint_paths": [ "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", "scripts/ops/hermes_handoff_catalog.py", "scripts/ops/hermes_handoff_checks_access.py", "scripts/ops/hermes_handoff_checks_delivery.py", @@ -150,24 +152,6 @@ ], "hygiene": { "max_lines": 500, - "legacy_line_exceptions": { - "scripts/tests/test_dashboards_render_atlas.py": { - "lines": 516, - "sha256": "afe4923391b157d03bcf0899208f77eeab79a2762da3575a365530b3074c5f25" - }, - "testing/tests/test_hermes_chat_quality.py": { - "lines": 2242, - "sha256": "71e9cafd14460b5132d07dd951514756d091009c23db7703db1575b05e34e75f" - }, - "testing/tests/test_hermes_cli_lanes.py": { - "lines": 1971, - "sha256": "992bf3788c33252e37a695f5307c31b1e8c0646e00a715e3e55f6901788abb62" - }, - "testing/tests/test_hermes_coordinator.py": { - "lines": 510, - "sha256": "290240cf391abc755eaa0abb12d247ad1718e7c9295a2092f349a7eaba4bfe55" - } - }, "line_limit_globs": [ "scripts/ops/hermes_handoff_*.py", "testing/**/*.py", @@ -221,6 +205,7 @@ "minimum_branch_percent": 95.0, "branch_tracked_files": [ "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", "scripts/ops/hermes_handoff_catalog.py", "scripts/ops/hermes_handoff_checks_access.py", "scripts/ops/hermes_handoff_checks_delivery.py", @@ -237,11 +222,11 @@ "scripts/ops/hermes_handoff_run.py", "ci/scripts/semgrep_report.py", "testing/quality_coverage.py", - "testing/quality_handoff_mutation.py", - "testing/quality_hygiene.py" + "testing/quality_handoff_mutation.py" ], "tracked_files": [ "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", "scripts/ops/hermes_handoff_catalog.py", "scripts/ops/hermes_handoff_checks_access.py", "scripts/ops/hermes_handoff_checks_delivery.py", diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py index a0c63fa0..95d2df42 100644 --- a/testing/quality_handoff_mutation.py +++ b/testing/quality_handoff_mutation.py @@ -138,6 +138,58 @@ MUTATIONS = ( "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" "assert e._closed_pr_problems(p,q)\n", ), + Mutation( + "absence-check-passes-on-zero-evidence", + "hermes_handoff_evaluators.py", + " if not observed:\n" + ' return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")\n', + " if False and not observed:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators as v\n" + "from hermes_handoff_exec import Outcome\nfrom hermes_handoff_model import *\n" + "s=CheckSpec('x','x','x','names_absent',steps=(Step('e','self',('kubectl',)),)," + "expect={'step':'e','names':('GITEA_TOKEN',)})\n" + "assert v.evaluate(s,{'e':Outcome(('x',),'x',0,'')}).status == NOT_RUN\n", + ), + Mutation( + "repository-pin-is-a-bare-prefix", + "hermes_handoff_policy.py", + ' and not path.startswith((f"{repo_root}/", f"{repo_root}?"))\n', + " and not path.startswith(repo_root)\n", + "from hermes_handoff_policy import *\n" + "try: check_argv((GITEA_CLIENT,'GET','/api/v1/repos/atlas/titan-iac-evil/pulls'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "exec-inner-command-is-not-rechecked", + "hermes_handoff_policy.py", + " check_argv(inner, mode)\n return\n", + " return\n", + "from hermes_handoff_policy import *\n" + "argv=('kubectl','exec','pod','--','/usr/local/bin/kubectl','get','ns','-o','name'," + "'--as','system:admin')\n" + "try: check_argv(argv)\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "unrecorded-step-loses-its-binary-attestation", + "hermes_handoff_run.py", + " executable_path=outcome.executable_path,\n" + " executable_sha256=outcome.executable_sha256,\n", + "", + "import hermes_handoff_run as r\nfrom hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "class V:\n name='self'\n" + "class N:\n" + " def run(self,argv,_v,_b=None): return Outcome(argv,'self',0,'locked'," + "executable_path='/usr/bin/git',executable_sha256='a'*64)\n" + "s=CheckSpec('x','x','x','stdout_matches'," + "steps=(Step('s','self',('git','status'),record=False),)," + "expect={'step':'s','equals':'locked'})\n" + "o=r.run_check(N(),s,{'self':V()}).outcomes[0]\n" + "assert o.executable_sha256 == 'a'*64 and o.executable_path == '/usr/bin/git'\n", + ), ) diff --git a/testing/quality_hygiene.py b/testing/quality_hygiene.py index 6b68de3a..5060ca02 100644 --- a/testing/quality_hygiene.py +++ b/testing/quality_hygiene.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import re from collections.abc import Iterable from pathlib import Path @@ -17,18 +16,6 @@ def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]: return sorted(matched) -def _is_unchanged_legacy( - config: dict[str, Any], root: Path, path: Path, line_count: int -) -> bool: - """Allow an over-cap legacy file only while its exact bytes stay unchanged.""" - relative = path.relative_to(root).as_posix() - record = config.get("legacy_line_exceptions", {}).get(relative) - if not isinstance(record, dict) or record.get("lines") != line_count: - return False - digest = hashlib.sha256(path.read_bytes()).hexdigest() - return record.get("sha256") == digest - - def run_check(contract: dict[str, Any], root: Path) -> list[str]: """Return human-readable issues for naming and file-size rules.""" config = contract.get("hygiene", {}) @@ -37,12 +24,8 @@ def run_check(contract: dict[str, Any], root: Path) -> list[str]: for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines and not _is_unchanged_legacy( - config, root, path, line_count - ): - issues.append( - f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})" - ) + if line_count > max_lines: + issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})") for rule in config.get("naming_rules", []): pattern = re.compile(rule["pattern"]) @@ -65,9 +48,7 @@ def count_files_over_line_limit(contract: dict[str, Any], root: Path) -> int: count = 0 for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines and not _is_unchanged_legacy( - config, root, path, line_count - ): + if line_count > max_lines: count += 1 return count diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py index 4f89db35..de4a411c 100644 --- a/testing/tests/test_hermes_handoff_acceptance.py +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -124,7 +124,6 @@ def test_the_parser_exposes_the_documented_controls() -> None: "21", "--pool-worker-env", "HERMES_POOL_LEASE_SECONDS", - "--expect-telegram-sessions", ] ) @@ -134,7 +133,8 @@ def test_the_parser_exposes_the_documented_controls() -> None: assert targets.expected_suspensions == ("flux-system/parked",) assert targets.node_count == 21 assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) - assert targets.expect_telegram_sessions is True + assert not hasattr(arguments, "concurrency") + assert not hasattr(arguments, "expect_telegram_sessions") def test_defaults_fall_back_to_the_shipped_targets() -> None: diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py index 9f54601a..c1f993f8 100644 --- a/testing/tests/test_hermes_handoff_ephemeral.py +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -15,6 +15,7 @@ from testing.tests.test_hermes_handoff_support import ( ) ephemeral = load_handoff_module("hermes_handoff_ephemeral") +arming = load_handoff_module("hermes_handoff_arming") exec_module = load_handoff_module("hermes_handoff_exec") policy = load_handoff_module("hermes_handoff_policy") model = load_handoff_module("hermes_handoff_model") @@ -84,16 +85,16 @@ def linked_worktree(tmp_path: Path) -> Path: (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") (gitdir / "commondir").write_text("../..\n", encoding="utf-8") (gitdir / "HEAD").write_text( - f"ref: refs/heads/{ephemeral.EXPECTED_HEAD_REF}\n", encoding="utf-8" + f"ref: refs/heads/{arming.EXPECTED_HEAD_REF}\n", encoding="utf-8" ) - ref = common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF + ref = common / "refs" / "heads" / arming.EXPECTED_HEAD_REF ref.parent.mkdir(parents=True) ref.write_text(f"{HEAD}\n", encoding="utf-8") remote_ref = common / "refs" / "remotes" / "origin" / "main" remote_ref.parent.mkdir(parents=True) remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") (common / "config").write_text( - f'[remote "origin"]\n\turl = {ephemeral.EXPECTED_REMOTE_URL}\n', + f'[remote "origin"]\n\turl = {arming.EXPECTED_REMOTE_URL}\n', encoding="utf-8", ) return root @@ -152,7 +153,7 @@ def test_worktree_attestation_rejects_each_malformed_local_source( wrong_sha = linked_worktree(tmp_path / "sha") gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() - (common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF).write_text("0" * 40) + (common / "refs" / "heads" / arming.EXPECTED_HEAD_REF).write_text("0" * 40) roots.append(wrong_sha) wrong_main = linked_worktree(tmp_path / "main") gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) @@ -175,7 +176,7 @@ def test_worktree_attestation_rejects_each_malformed_local_source( oversized = tmp_path / "oversized" oversized.write_text("x" * 5, encoding="utf-8") with pytest.raises(ephemeral.ArmingError): - ephemeral._bounded_read(oversized, 4) + ephemeral.bounded_read(oversized, 4) def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: @@ -253,26 +254,24 @@ def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> Non assert {"number is malformed", "draft does not match", "base is malformed"} <= set( problems ) - assert ephemeral._discover(exec_module.Outcome((), "x", returncode=1), request())[1] - for body in ( - "not-json", - "{}", - "[null]", - json.dumps([{"head": {"ref": "other"}, "base": {}}]), - ): - numbers, error = ephemeral._discover( + assert ephemeral._created_number( + exec_module.Outcome((), "x", returncode=1), request() + ) == (0, "pull-request creation failed: exit status 1") + for body in ("not-json", "{}"): + number, error = ephemeral._created_number( exec_module.Outcome((), "x", 0, body), request() ) + assert number == 0 and error + assert ephemeral._created_number( + exec_module.Outcome((), "x", 0, json.dumps(pr_payload())), request() + ) == (42, "") + for rows in ([None], [{"head": {"ref": "other"}, "base": {}}]): + numbers, error = ephemeral._match_pulls(rows, request()) assert numbers == [] and error invalid = pr_payload() invalid["draft"] = False - assert ephemeral._discover( - exec_module.Outcome((), "x", 0, json.dumps([invalid])), request() - )[1] - assert ( - "discovered 0" - in ephemeral._discover(exec_module.Outcome((), "x", 0, "[]"), request())[1] - ) + assert ephemeral._match_pulls([invalid], request())[1] + assert "discovered 0" in ephemeral._match_pulls([], request())[1] @pytest.mark.parametrize( @@ -335,3 +334,105 @@ def test_read_only_runner_cannot_reach_push() -> None: results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) assert results["ephemeral.feature-branch-push"].status == model.FAIL assert not any(call["argv"][1] == "push" for call in spawn.calls) + + +def push_prelude(ref_line: str) -> list: + """The four scripted results that get an armed run to a verified push.""" + return [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + ] + + +def test_pull_request_discovery_reads_every_page() -> None: + """A full first page must not end discovery: the match can be on page two.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + filler = [ + { + "number": index, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/other", "sha": HEAD}, + } + for index in range(1, ephemeral.DISCOVERY_PAGE_SIZE + 1) + ] + runner, spawn = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps(filler)), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.draft-pull-request"].status == model.PASS + pages = [call["argv"][2] for call in spawn.calls if "pulls?state=all" in str(call)] + assert any("page=1" in page for page in pages) + assert any("page=2" in page for page in pages) + + +def test_uncertain_discovery_still_closes_the_created_pull_and_names_residue() -> None: + """The create response names the pull, so an unusable index cannot orphan it.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(returncode=1), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + draft = results["ephemeral.draft-pull-request"] + + assert draft.status == model.FAIL + assert draft.evidence["pull_requests"] == [42] + assert draft.evidence["residue_ref"] == request().ref + assert any("pulls/42" in line for line in draft.evidence["manual_cleanup"]) + assert any("branches/" in line for line in draft.evidence["manual_cleanup"]) + # The pull the harness created is closed, not left behind. + assert any( + call["argv"][1] == "PATCH" and call["argv"][2].endswith("/pulls/42") + for call in spawn.calls + ) + assert results["ephemeral.cleanup-verified"].status == model.FAIL + + +def test_a_wholly_unidentifiable_create_names_the_ref_for_manual_cleanup() -> None: + """When neither source names a pull, the operator gets the exact ref to sweep.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout="not-json"), + completed(returncode=1), + completed(), + completed(), + ] + ) + + draft = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.draft-pull-request" + ] + + assert draft.status == model.FAIL + assert draft.evidence["pull_requests"] == [] + assert draft.evidence["residue_ref"] == request().ref + assert any( + request().ref in line and "close by hand" in line + for line in draft.evidence["manual_cleanup"] + ) diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py index c192448d..fb54d3a5 100644 --- a/testing/tests/test_hermes_handoff_exec.py +++ b/testing/tests/test_hermes_handoff_exec.py @@ -226,8 +226,6 @@ def test_combined_output_joins_only_the_streams_that_carry_text() -> None: {"deadline_seconds": -1}, {"max_bytes": -1}, {"max_bytes": 10**100}, - {"concurrency": 0}, - {"concurrency": 17}, ], ) def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( @@ -255,7 +253,6 @@ def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: ("git", "fetch", "origin"), ("git", "config", "core.sshCommand", "x"), ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), - ("helm", "get", "values", "x"), ] for command in commands: assert runner.run(command, OPERATOR).error.startswith("policy:") diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py index 1e84cfb8..61c60705 100644 --- a/testing/tests/test_hermes_handoff_mutation_gate.py +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -4,7 +4,13 @@ from __future__ import annotations from pathlib import Path -from testing.quality_handoff_mutation import Mutation, MutationReport, main, run_gate +from testing.quality_handoff_mutation import ( + MUTATIONS, + Mutation, + MutationReport, + main, + run_gate, +) ROOT = Path(__file__).parents[2] @@ -38,8 +44,11 @@ def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: def test_main_reports_a_perfect_score(capsys) -> None: + """Every mutant must die, and the count must track the declared set.""" assert main() == 0 - assert "9/9 (100.00%)" in capsys.readouterr().out + total = len(MUTATIONS) + assert total >= 13 + assert f"{total}/{total} (100.00%)" in capsys.readouterr().out def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py index 90cef791..41f1fc7e 100644 --- a/testing/tests/test_hermes_handoff_policy.py +++ b/testing/tests/test_hermes_handoff_policy.py @@ -9,8 +9,8 @@ from testing.tests.test_hermes_handoff_support import load_handoff_module policy = load_handoff_module("hermes_handoff_policy") -def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: - policy.check_argv(argv, mode or policy.READ_ONLY, allow_impersonation=inner) +def check(*argv: str, mode: str | None = None) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY) @pytest.mark.parametrize( @@ -26,8 +26,6 @@ def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: "-o", policy.projection("jsonpath={.status.phase}"), ), - ("flux", "get", "kustomizations", "--all-namespaces"), - ("helm", "list", "--all-namespaces"), ("git", "merge-base", "--is-ancestor", "a", "b"), ("git", "ls-remote", "origin", "refs/heads/main"), ("hermes", "kanban", "list", "--json"), @@ -63,8 +61,10 @@ def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: ("kubectl", "exec", "pod"), ("kubectl", "exec", "pod", "--"), ("kubectl",), + ("flux", "get", "kustomizations", "--all-namespaces"), ("flux", "reconcile", "kustomization", "x"), ("flux",), + ("helm", "list", "--all-namespaces"), ("helm", "upgrade", "x"), ("helm", "get", "values", "x"), ("git", "commit", "-m", "x"), @@ -103,22 +103,41 @@ def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: check("kubectl", "delete", "-n", "get", "configmap", "x") -def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: - with pytest.raises(policy.PolicyError, match="forbidden argument"): - check("kubectl", "get", "namespaces", "--as", "system:admin") - check( +@pytest.mark.parametrize("flag", ["--as", "--as-group", "--as-uid"]) +@pytest.mark.parametrize("mode", ["read-only", "ephemeral-armed"]) +def test_impersonation_is_refused_from_every_vantage_and_mode( + flag: str, mode: str +) -> None: + """No vantage may impersonate: not the operator, and not inside a pod. + + The inner command of ``kubectl exec`` used to be granted impersonation + unconditionally, so a self/switchyard/node/chat step carrying ``--as`` was + accepted. The inner argv is now re-checked under the same rule. + """ + for argv in ( + ("kubectl", "get", "namespaces", "-o", "name", flag, "system:admin"), + ("kubectl", "get", "ns", "-o", "name", f"{flag}=system:admin"), + ): + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check(*argv, mode=mode) + inner = ( "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", "get", - "namespaces", + "ns", "-o", "name", - "--as", + flag, "system:admin", - inner=True, ) + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check(*inner, mode=mode) -def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: +def test_exec_validates_its_inner_command() -> None: script = policy.render_shell("env_names") policy._RENDERED.add(script) check( @@ -134,13 +153,42 @@ def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() - "ns", "-o", "name", - "--as", - "system:admin", ) with pytest.raises(policy.PolicyError): check("kubectl", "exec", "pod", "--", "bash", "-c", "id") +def test_the_repository_pin_has_an_exact_boundary_and_rejects_dot_segments() -> None: + """A prefix match without a boundary let a look-alike repository through.""" + for path in ( + "/api/v1/repos/atlas/titan-iac-evil/pulls", + "/api/v1/repos/atlas/titan-iacx", + "/api/v1/repos/atlas/titan-iac../pulls", + ): + with pytest.raises(policy.PolicyError): + check(policy.GITEA_CLIENT, "GET", path) + for path in ( + "/api/v1/repos/atlas/titan-iac/../../user/tokens", + "/api/v1/repos/atlas/titan-iac/%2e%2e/%2e%2e/user/tokens", + "/api/v1/repos/atlas/titan-iac/%252e%252e/user/tokens", + "/api/v1/repos/atlas/titan-iac/./pulls", + "/api/v1/repos/atlas/titan-iac//pulls", + ): + with pytest.raises(policy.PolicyError, match="relative segments"): + check(policy.GITEA_CLIENT, "GET", path) + check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac") + check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac/pulls?state=all") + + +def test_every_allowlisted_binary_is_release_attestable() -> None: + """An allowlisted binary with no pinned digest is dead, misleading config.""" + exec_module = load_handoff_module("hermes_handoff_exec") + assert set(exec_module.EXPECTED_SHA256) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.EXPECTED_PATHS) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.POD_COMMAND_PATHS) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.CONTEXT_AWARE_BINARIES) <= set(policy.ALLOWED_BINARIES) + + def test_shell_templates_reject_credential_roots() -> None: with pytest.raises(policy.PolicyError, match="credential paths"): policy.shell("json_fields", path="/runtime-access/token", fields="state") diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py index 881ffd55..4d87fb3e 100644 --- a/testing/tests/test_hermes_handoff_run.py +++ b/testing/tests/test_hermes_handoff_run.py @@ -135,7 +135,6 @@ def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> N deployment_revision="0", node_count=1, pool_replicas=1, - expect_telegram_sessions=False, chat_config_revision="", pool_worker_env=(), dependency_heads=((14, "bad"), (14, "bad")), @@ -335,6 +334,25 @@ def test_a_check_whose_vantage_is_absent_is_not_run() -> None: assert result.outcomes[0].error == "vantage node is unavailable" +def test_validate_catalog_rejects_impersonation_from_every_vantage() -> None: + """The operator vantage was the only one guarded; now none is exempt.""" + for vantage in (catalog.OPERATOR, catalog.SELF, catalog.CHAT): + spec = check( + "stdout_matches", + {"step": "s", "equals": "x"}, + steps=( + model.Step( + key="s", + vantage=vantage, + argv=("kubectl", "get", "ns", "-o", "name", "--as", "system:admin"), + ), + ), + ) + problems = harness_run.validate_catalog([spec]) + assert any("impersonates" in problem for problem in problems) + assert any(vantage in problem for problem in problems) + + def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: spec = check( "stdout_matches", @@ -357,6 +375,13 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None assert result.status == model.PASS assert result.outcomes[0].stdout == harness_run.UNRECORDED assert result.outcomes[0].stderr == harness_run.UNRECORDED + # Withholding bulk bytes must not withhold the binary that produced them. + recorded = result.outcomes[0] + assert recorded.executable_path + assert recorded.executable_sha256 + assert recorded.executable_path == "kubectl" + assert recorded.executable_sha256 == "a" * 64 + assert recorded.as_dict()["executable_sha256"] == recorded.executable_sha256 def test_an_expired_deadline_leaves_the_report_saying_so() -> None: diff --git a/testing/tests/test_hermes_handoff_zero_evidence.py b/testing/tests/test_hermes_handoff_zero_evidence.py new file mode 100644 index 00000000..8e47768a --- /dev/null +++ b/testing/tests/test_hermes_handoff_zero_evidence.py @@ -0,0 +1,243 @@ +"""Regressions for the zero-evidence fail-open in the absence checks. + +``evaluate_names_absent`` used to return ``PASS`` when its step exited 0 with no +output, so five mandatory checks — the ones asserting that provider API keys, +forge credentials, cluster-admin bindings, and shared coordinator state are +*absent* — could report a pass on no evidence at all and turn a ``NO_GO`` into a +``GO``. These tests pin the real catalog specs, not synthetic look-alikes, and +reproduce both ways silence actually arrives: a POSIX pipeline whose exit status +comes from its last stage, and a ``kubectl -o jsonpath`` over drifted structure. +""" + +from __future__ import annotations + +import datetime +import shutil +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome + +catalog = load_handoff_module("hermes_handoff_catalog") +evaluators = load_handoff_module("hermes_handoff_evaluators") +harness_run = load_handoff_module("hermes_handoff_run") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") + +# Every mandatory check whose whole claim is that something is not there. +ABSENCE_CHECKS = ( + "access.no-cluster-admin-binding-for-agent", + "forge.worker-holds-no-git-credential-variable", + "identity.no-provider-api-key-in-manifest", + "identity.no-provider-api-key-in-pod", + "pool.coordinator-retains-sole-state-ownership", +) + + +def real_catalog() -> dict: + targets = catalog.Targets(now=datetime.datetime.now(datetime.UTC)) + return {spec.id: spec for spec in harness_run.build_catalog(targets)} + + +def test_the_five_absence_checks_are_exactly_the_mandatory_names_absent_set() -> None: + """If a sixth absence check appears it must be covered here too.""" + specs = real_catalog() + found = { + identifier + for identifier, spec in specs.items() + if spec.rule == "names_absent" and spec.mandatory + } + assert found == set(ABSENCE_CHECKS) + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_never_passes_on_zero_observations( + identifier: str, +) -> None: + """rc=0 with no lines is drift, so the verdict must be NOT_RUN, not PASS.""" + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + + silent = evaluators.evaluate(spec, {step_key: outcome(stdout="")}) + assert silent.status == model.NOT_RUN + assert "no lines" in silent.reason + assert silent.status != model.PASS + + whitespace = evaluators.evaluate(spec, {step_key: outcome(stdout="\n \n\t\n")}) + assert whitespace.status == model.NOT_RUN + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_still_passes_on_real_evidence(identifier: str) -> None: + """The guard must not turn a healthy observation into a false NOT_RUN.""" + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + benign = "clusterrolebinding.rbac.authorization.k8s.io/system:basic-user\nPATH\n" + + passed = evaluators.evaluate(spec, {step_key: outcome(stdout=benign)}) + assert passed.status == model.PASS + assert passed.evidence["observed_count"] == 2 + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_still_fails_on_a_forbidden_name(identifier: str) -> None: + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + offender = next( + iter(spec.expect.get("names", ()) or spec.expect.get("contains", ())) + ) + + verdict = evaluators.evaluate( + spec, {step_key: outcome(stdout=f"OTHER\n{offender}")} + ) + assert verdict.status == model.FAIL + assert offender in verdict.reason + + +def test_zero_evidence_on_a_mandatory_absence_check_forces_no_go() -> None: + """A NOT_RUN on a mandatory check must not be rounded up to a GO. + + Every other check is held at PASS so the verdict turns on the five absence + checks alone: silence must cost the release its GO. + """ + specs = real_catalog() + silent, green = [], [] + for identifier, spec in specs.items(): + status = model.PASS + if identifier in ABSENCE_CHECKS: + evaluation = evaluators.evaluate( + spec, {spec.expect["step"]: outcome(stdout="")} + ) + assert evaluation.status == model.NOT_RUN + status = evaluation.status + silent.append(model.CheckResult(spec=spec, status=status)) + green.append(model.CheckResult(spec=spec, status=model.PASS)) + + assert all(result.blocking for result in silent if result.status == model.NOT_RUN) + assert model.Report(mode="read-only", started_at="t", results=silent).decision == ( + model.NO_GO + ) + assert ( + model.Report(mode="read-only", started_at="t", results=green).decision + == model.GO + ) + + +def test_a_broken_shell_pipeline_exits_zero_with_no_output() -> None: + """The harness' own frozen template is reachable with rc=0 and 0 bytes. + + A POSIX pipeline reports the status of its *last* stage, so a missing or + renamed upstream binary produces success-with-no-evidence rather than an + error. This is one of the two live paths that made the fail-open reachable. + """ + script = policy.render_shell("env_names").replace("/usr/bin/env", "/usr/bin/absent") + + completed = subprocess.run( # noqa: S603 + ["/bin/sh", "-c", script], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert completed.returncode == 0 + assert completed.stdout == "" + + spec = real_catalog()["identity.no-provider-api-key-in-pod"] + verdict = evaluators.evaluate( + spec, {"env": outcome(stdout=completed.stdout, returncode=completed.returncode)} + ) + assert verdict.status == model.NOT_RUN + + +DRIFTED_JSONPATH_OUTPUT = { + # A renamed parent — `containerz` — makes kubectl print nothing at all. + "renamed parent": "", + # A renamed leaf — `.nam` — makes it print only the separators. + "renamed leaf": "\n" * 6, + # A container filter that matches nothing behaves the same way. + "unmatched filter": "", +} + + +@pytest.mark.parametrize("shape", sorted(DRIFTED_JSONPATH_OUTPUT)) +def test_drifted_kubectl_jsonpath_output_is_not_run(shape: str) -> None: + """The exact bytes a drifted ``-o jsonpath`` emits must never read as a pass. + + ``kubectl -o jsonpath`` exits 0 when the structure it walks is missing, so a + renamed field, a renamed leaf, or an unmatched container filter all deliver + success with no usable evidence. Pinned as literal output so the regression + holds with no cluster and no network; the live rc=0 behaviour is asserted + separately by :func:`test_a_live_drifted_projection_still_exits_zero`. + """ + spec = real_catalog()["identity.no-provider-api-key-in-manifest"] + + verdict = evaluators.evaluate( + spec, {"env": outcome(stdout=DRIFTED_JSONPATH_OUTPUT[shape])} + ) + + assert verdict.status == model.NOT_RUN + assert "no lines" in verdict.reason + + +def test_a_live_drifted_projection_still_exits_zero() -> None: + """Confirm against the real tool that drift is success, not an error. + + kubectl has no offline JSONPath renderer, so this reads the live agent + workload — a plain, read-only ``get`` with a name-only projection — and skips + when no cluster is reachable. The hermetic assertion above does not skip. + """ + kubectl = shutil.which("kubectl") + if kubectl is None: # pragma: no cover - kubectl ships with the release image + pytest.skip("kubectl is not installed in this environment") + base = ["--namespace", "hermes", "get", "deploy/hermes-agent", "-o"] + healthy = 'jsonpath={range .spec.template.spec.containers[*]}{.name}{"\\n"}{end}' + probe = subprocess.run( # noqa: S603 + [kubectl, *base, healthy], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if probe.returncode != 0: + pytest.skip("no read-only cluster access in this environment") + assert probe.stdout.strip() + + for drifted in ( + 'jsonpath={range .spec.template.spec.containerz[*]}{.name}{"\\n"}{end}', + 'jsonpath={range .spec.template.spec.containers[*]}{.nam}{"\\n"}{end}', + ): + result = subprocess.run( # noqa: S603 + [kubectl, *base, drifted], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "" + spec = real_catalog()["identity.no-provider-api-key-in-manifest"] + verdict = evaluators.evaluate(spec, {"env": outcome(stdout=result.stdout)}) + assert verdict.status == model.NOT_RUN + + +def test_every_absence_probe_yields_one_line_per_observed_item() -> None: + """A healthy probe must produce evidence, or the guard would misfire. + + The pool claim projection emits ``=`` so a template volume + without a PVC still counts as an observation; an empty result therefore only + happens when the object or the projection has drifted. + """ + spec = real_catalog()["pool.coordinator-retains-sole-state-ownership"] + argv = spec.steps[0].argv + assert '{.name}{"="}' in argv[-1] + + verdict = evaluators.evaluate(spec, {"claims": outcome(stdout="tmp=\nconfig=\n")}) + assert verdict.status == model.PASS + assert verdict.evidence["observed_count"] == 2 + + leaked = evaluators.evaluate( + spec, {"claims": outcome(stdout="state=hermes-agent-home\n")} + ) + assert leaked.status == model.FAIL diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index fecae836..3c9ab2e3 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -2,17 +2,13 @@ from __future__ import annotations -import hashlib from pathlib import Path import textwrap from testing.quality_contract import load_contract from testing.quality_coverage import run_check as run_coverage_check from testing.quality_docs import run_check as run_docs_check -from testing.quality_hygiene import ( - count_files_over_line_limit, - run_check as run_hygiene_check, -) +from testing.quality_hygiene import run_check as run_hygiene_check def test_bundled_contract_exposes_local_and_jenkins_profiles(): @@ -28,7 +24,7 @@ def test_handoff_modules_are_exactly_managed_linted_and_covered(): expected = { path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") } - assert len(expected) == 15 + assert len(expected) == 16 assert expected <= set(contract["managed_modules"]) assert expected <= set(contract["lint_paths"]) assert expected <= set(contract["coverage"]["tracked_files"]) @@ -112,31 +108,6 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): ) -def test_hygiene_legacy_exception_is_exact_and_invalidated_by_any_edit(tmp_path: Path): - legacy = tmp_path / "legacy.py" - legacy.write_text("line\n" * 4, encoding="utf-8") - contract = { - "hygiene": { - "max_lines": 3, - "line_limit_globs": ["*.py"], - "legacy_line_exceptions": { - "legacy.py": { - "lines": 4, - "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), - } - }, - } - } - assert run_hygiene_check(contract, tmp_path) == [] - assert count_files_over_line_limit(contract, tmp_path) == 0 - - legacy.write_text("edit\n" * 4, encoding="utf-8") - assert run_hygiene_check(contract, tmp_path) == [ - "file exceeds 3 LOC: legacy.py (4)" - ] - assert count_files_over_line_limit(contract, tmp_path) == 1 - - def test_coverage_check_enforces_per_file_floor(tmp_path: Path): build_dir = tmp_path / "build" build_dir.mkdir() From 02e559d7ac80cde6b2f2b87a4cba32bcb79d8971 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:27:28 +0000 Subject: [PATCH 19/46] docs(hermes): record the recomputed conflict graph and head SHAs Refresh #12 and #18 to their live Gitea heads, record #19's replaced head exactly, and replace the pairwise conflict set with a freshly computed one over all 28 combinations. #18+#19 is now clean because #19 no longer duplicates the repo-wide hygiene legacy-exception mechanism that #18 also carries; #12+#19 is new because #12 advanced. Name the exact overlapping files per conflicting pair and how to resolve each additively. Co-Authored-By: Claude Opus 5 --- docs/hermes_full_handoff_acceptance.md | 50 +++++++++++++++++--------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index 5533ef2d..23efe79f 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -197,22 +197,29 @@ remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: | PR | Head SHA | Draft | Base | Role | | --- | --- | --- | --- | --- | -| #12 | `465d386da5237f011cb408f1a3d4b7d0c6d8ffae` | no | `main` | AI usage exporter repair | +| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair | | #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | | #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | | #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | | #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | -| #18 | `20002527512235b29054f3636b02418168373009` | yes | `main` | distributed three-worker pool | -| #19 | `8f005458282269ba5c07941814e4237f5d4cf3ac` | yes | `main` | reviewed blocked head; this repair replaces it on the same branch | +| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool | +| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch | | #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | The repaired #19 SHA cannot be embedded in its own commit without changing that SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; -the PR handoff records the exact value. All eight snapshot heads merge cleanly -with the snapshot `main` individually. Pairwise `git merge-tree --write-tree` -found conflicts only for: #12+#17, #14+#15, #14+#16, #14+#17, #14+#20, -#15+#16, #15+#17, #16+#20, and #17+#20. Gitea's `mergeable=false` on a draft is -not treated as conflict evidence. +the PR handoff records the exact value. All eight heads in this table, including +the repaired #19, merge cleanly with `main` individually. Pairwise +`git merge-tree --write-tree` over all 28 combinations found conflicts for: +#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16, +#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's +`mergeable=false` on a draft is not treated as conflict evidence. + +This graph differs from the previous snapshot in exactly two ways. #12 and #18 +both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer +duplicates the repo-wide hygiene legacy-exception mechanism that #18 also +carries, which makes #18+#19 clean. Recompute before every integration step; +nothing here is durable. Use this dependency order, refreshing every head and re-running merge-tree at each step: @@ -227,16 +234,25 @@ each step: 6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are merged and their exact supplied heads are ancestors of refreshed `main`. -#19 overlaps `testing/quality_contract.json` with #14/#15/#16, -`testing/quality_coverage.py` and its helper tests with #14, -`test_hermes_auto_router.py` with #18, and existing Kustomize content with -#12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, -and all desired-state changes during conflict resolution. +An exact post-repair merge-tree check reports #19 clean with current `main`, #17, +#18, and #20, and conflicting with #12, #14, #15, and #16 on these files: -An exact post-repair merge-tree check against the heads in the table reports -#19 clean with current `main`, #12, #17, and #20, and conflicting with #14, -#15, #16, and #18. Recompute this graph from the Gitea API immediately before -integration; do not infer conflict status from Gitea's draft `mergeable` field. +| Pair | Overlapping files | +| --- | --- | +| #12+#19 | `testing/tests/test_hermes_auto_router.py` | +| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` | +| #15+#19 | `testing/tests/conftest.py` | +| #16+#19 | `testing/quality_contract.json` | + +Every one of those overlaps is additive on the #19 side. Resolve the contract by +union — take every PR's managed modules, lint paths, tracked files, and branch +tracked files, and never take one PR's quality contract wholesale. #19 adds two +fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test; +keep both sides. Preserve branch coverage, every tracked module, fixture +isolation, and all desired-state changes during conflict resolution. + +Recompute this graph from the Gitea API immediately before integration; do not +infer conflict status from Gitea's draft `mergeable` field. The #16/#19 integration must keep both the immutable digest and the source tag (`newTag: git--build-` plus the digest) in rendered image identity. From 7a55b259bf1224d9bf19f5ec294f3c917fb4bfc8 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:31:15 +0000 Subject: [PATCH 20/46] hermes: add the fenced three-node distributed execution pool Three fenced worker Pods claim Hermes Kanban runs through a coordinator that owns every state transition, with per-ordinal HMAC authority, a mediated broker-only SCM path, and durable per-ordinal workspaces. Content is the reviewed head of PR #18 (689bcb6e) with PR 16's and PR 19's contributions removed: they were merged in only to validate co-existence and are not prerequisites, so this branch no longer carries them as ancestors. Only PR 14 and PR 15 remain, because the broker boundary and the cli_lane_* decomposition are load-bearing for two of the fixed P0 boundaries. Co-Authored-By: Claude Opus 5 --- ci/scripts/semgrep_report.py | 2 +- .../applications/hermes/kustomization.yaml | 4 + ...test_dashboards_render_atlas_drilldowns.py | 2 +- services/hermes-scm-broker/deployment.yaml | 3 +- services/hermes-scm-broker/networkpolicy.yaml | 6 + services/hermes/NOTES.md | 26 + services/hermes/agent-deployment.yaml | 2 +- .../hermes/execution-coordinator-patch.yaml | 66 +++ services/hermes/execution-mediator.yaml | 425 +++++++++++++++ .../execution-worker-networkpolicy.yaml | 206 ++++++++ services/hermes/execution-worker-rbac.yaml | 12 + .../hermes/execution-worker-statefulset.yaml | 264 ++++++++++ services/hermes/kustomization.yaml | 37 ++ services/hermes/scm-common/kustomization.yaml | 2 +- .../hermes/scm-common/scripts/scm_broker.py | 41 +- services/hermes/scripts/cli_lane_dispatch.py | 25 +- services/hermes/scripts/cli_lane_evidence.py | 15 +- services/hermes/scripts/cli_lane_execution.py | 26 +- .../hermes/scripts/cli_lane_finalization.py | 1 + .../hermes/scripts/cli_lane_quarantine.py | 2 + services/hermes/scripts/cli_lane_retention.py | 9 + .../hermes/scripts/execution_pool_client.py | 221 ++++++++ .../scripts/execution_pool_coordinator.py | 473 +++++++++++++++++ .../hermes/scripts/execution_pool_project.py | 152 ++++++ .../hermes/scripts/execution_pool_protocol.py | 496 ++++++++++++++++++ services/hermes/scripts/execution_pool_scm.py | 281 ++++++++++ .../hermes/scripts/execution_pool_server.py | 130 +++++ .../hermes/scripts/execution_pool_worker.py | 445 ++++++++++++++++ .../hermes/scripts/stage_runtime_access.py | 177 ++++++- services/hermes/service.yaml | 31 ++ .../vault/scripts/vault_k8s_auth_configure.sh | 2 + testing/tests/test_hermes_agent_access.py | 32 +- testing/tests/test_hermes_agent_security.py | 67 ++- testing/tests/test_hermes_auto_router.py | 1 + testing/tests/test_hermes_chat_config.py | 11 +- .../test_hermes_chat_provider_runtime.py | 196 +++++++ testing/tests/test_hermes_chat_voice.py | 7 +- testing/tests/test_hermes_cli_capabilities.py | 8 +- .../tests/test_hermes_cli_dispatch_runtime.py | 37 ++ .../tests/test_hermes_cli_evidence_edges.py | 35 ++ .../tests/test_hermes_cli_execution_edges.py | 15 +- testing/tests/test_hermes_cli_fallback.py | 4 + .../test_hermes_cli_finalization_edges.py | 8 + .../test_hermes_cli_foundation_coverage.py | 60 ++- .../test_hermes_cli_lanes_configuration.py | 12 +- testing/tests/test_hermes_cli_lanes_kanban.py | 392 -------------- .../tests/test_hermes_cli_provider_edges.py | 67 ++- .../tests/test_hermes_cli_records_edges.py | 11 + .../tests/test_hermes_cli_recovery_edges.py | 34 ++ .../tests/test_hermes_cli_retention_edges.py | 46 ++ .../tests/test_hermes_coordinator_boards.py | 9 +- testing/tests/test_hermes_execution_pool.py | 476 +++++++++++++++++ .../test_hermes_execution_pool_assignment.py | 208 ++++++++ ...st_hermes_execution_pool_coordinator_v2.py | 293 +++++++++++ .../test_hermes_execution_pool_dispatch_v2.py | 410 +++++++++++++++ .../test_hermes_execution_pool_mediator.py | 447 ++++++++++++++++ .../test_hermes_execution_pool_project.py | 211 ++++++++ .../test_hermes_execution_pool_protocol_v2.py | 241 +++++++++ ...est_hermes_execution_pool_scm_tampering.py | 200 +++++++ ...st_hermes_execution_pool_worker_execute.py | 246 +++++++++ .../test_hermes_execution_pool_worker_v2.py | 292 +++++++++++ .../tests/test_hermes_gitea_pr_integration.py | 2 +- ...est_hermes_node_account_privilege_audit.py | 6 +- testing/tests/test_hermes_runtime_access.py | 88 +++- .../test_hermes_runtime_stage_coverage.py | 96 +++- testing/tests/test_hermes_scm_broker.py | 11 +- 66 files changed, 7344 insertions(+), 519 deletions(-) create mode 100644 services/hermes/execution-coordinator-patch.yaml create mode 100644 services/hermes/execution-mediator.yaml create mode 100644 services/hermes/execution-worker-networkpolicy.yaml create mode 100644 services/hermes/execution-worker-rbac.yaml create mode 100644 services/hermes/execution-worker-statefulset.yaml create mode 100644 services/hermes/scripts/execution_pool_client.py create mode 100644 services/hermes/scripts/execution_pool_coordinator.py create mode 100644 services/hermes/scripts/execution_pool_project.py create mode 100644 services/hermes/scripts/execution_pool_protocol.py create mode 100644 services/hermes/scripts/execution_pool_scm.py create mode 100644 services/hermes/scripts/execution_pool_server.py create mode 100644 services/hermes/scripts/execution_pool_worker.py create mode 100644 testing/tests/test_hermes_chat_provider_runtime.py delete mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_execution_pool.py create mode 100644 testing/tests/test_hermes_execution_pool_assignment.py create mode 100644 testing/tests/test_hermes_execution_pool_coordinator_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_dispatch_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_mediator.py create mode 100644 testing/tests/test_hermes_execution_pool_project.py create mode 100644 testing/tests/test_hermes_execution_pool_protocol_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_scm_tampering.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_execute.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_v2.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..2408f3e9 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -52,7 +52,7 @@ def build_report( results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index df7a5560..084e6074 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -30,6 +30,10 @@ spec: kind: Deployment name: hermes-agent namespace: hermes + - apiVersion: apps/v1 + kind: StatefulSet + name: hermes-execution-worker + namespace: hermes - apiVersion: apps/v1 kind: DaemonSet name: hermes-node-ssh-access diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py index 08871b71..904379c9 100644 --- a/scripts/tests/test_dashboards_render_atlas_drilldowns.py +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -1,6 +1,6 @@ """Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" -from scripts.tests.test_dashboards_render_atlas import load_module +from scripts.tests.test_dashboard_render_support import load_module def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml index 649ada1d..ba62f451 100644 --- a/services/hermes-scm-broker/deployment.yaml +++ b/services/hermes-scm-broker/deployment.yaml @@ -17,6 +17,7 @@ spec: labels: app: hermes-scm-broker annotations: + ai.bstein.dev/config-rev: scm-boundary-v2 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-scm-broker vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea @@ -91,7 +92,7 @@ spec: volumes: - name: broker-code configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: tmp emptyDir: diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml index 8187e333..9c1a0542 100644 --- a/services/hermes-scm-broker/networkpolicy.yaml +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -17,6 +17,12 @@ spec: podSelector: matchLabels: app: hermes-agent + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-execution-mediator ports: - {protocol: TCP, port: 9081} egress: diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..400bf44f 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -257,6 +257,32 @@ Use this short explanation: leading different real incidents, improving the skill after failures, and teaching the architecture without prompts. +## Distributed execution-pool rollout boundary + +- Do not reconcile the pool until the approved credential owner has provisioned + six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json` + and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an + independent account or refresh-token lineage; copying one rotating refresh + token into multiple fields recreates the lost-update failure this design + prevents. +- Worker credential files live on separate ordinal-owned RWO `provider-access` + claims. Provider refresh updates these durable private copies across Pod + restarts; they are never synchronized back to Vault. Rotate one bootstrap + credential at a time through the reviewed Vault workflow and reinitialize + only that ordinal after human approval. +- Existing task worktrees remain on the single local owner lane. Only tasks + without `workspace_path` enter the distributed pool, where each ordinal owns + one RWO checkout. Repository and base-branch identity comes from the canonical + board registry rather than task-supplied metadata. +- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token. + Ordinal mediator Deployments are separate network identities colocated with + the matching workspace PVC; they alone authenticate exact-run results and + reach the PR14 SCM broker. +- A hashed execution-pool ConfigMap, protocol-version readiness checks, and + versioned SCM boundary name make code/config changes controlled rollouts. A + rollout is still a human-reviewed operation; this repository change does not + reconcile or deploy it. + ## Your shortest path to fluency 1. Explain the request diagram without looking. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ccbc36ea..ed0533e7 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -1191,7 +1191,7 @@ spec: - {key: openai.yaml, path: agents/openai.yaml} - name: scm-boundary configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: image-policy configMap: diff --git a/services/hermes/execution-coordinator-patch.yaml b/services/hermes/execution-coordinator-patch.yaml new file mode 100644 index 00000000..ec9ab80b --- /dev/null +++ b/services/hermes/execution-coordinator-patch.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-agent + namespace: hermes +spec: + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + spec: + containers: + - name: cli-lane-runner + env: + - {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"} + - {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"} + - name: execution-pool-coordinator + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py] + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HOME, value: /opt/data/home} + - {name: PYTHONPATH, value: /opt/hermes} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true} + - {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + - {name: tmp, mountPath: /tmp} + ports: + - {name: execution-pool, containerPort: 9007, protocol: TCP} + startupProbe: + httpGet: {path: /ready, port: execution-pool} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: execution-pool} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /ready, port: execution-pool} + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: 500m, memory: 512Mi} + volumes: + - name: execution-pool-code + configMap: + name: hermes-execution-pool + defaultMode: 0555 diff --git a/services/hermes/execution-mediator.yaml b/services/hermes/execution-mediator.yaml new file mode 100644 index 00000000..d41ee9f9 --- /dev/null +++ b/services/hermes/execution-mediator.yaml @@ -0,0 +1,425 @@ +# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is +# colocated with exactly one ordinal's RWO workspace but has a distinct network +# identity, private HMAC derivation, and durable integrity state. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-0, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-1, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-2, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-0, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-1, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-2, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-0 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-0 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-1 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-1 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-2 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-2 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] diff --git a/services/hermes/execution-worker-networkpolicy.yaml b/services/hermes/execution-worker-networkpolicy.yaml new file mode 100644 index 00000000..93e46be8 --- /dev/null +++ b/services/hermes/execution-worker-networkpolicy.yaml @@ -0,0 +1,206 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + policyTypes: [Ingress, Egress] + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + podSelector: + matchLabels: {k8s-app: kube-dns} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: vault} + podSelector: + matchLabels: {app: vault} + ports: [{protocol: TCP, port: 8200}] + - to: + - podSelector: + matchLabels: {app: hermes-switchyard} + ports: [{protocol: TCP, port: 9005}] + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 224.0.0.0/4 + ports: [{protocol: TCP, port: 443}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-0 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-1 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-2 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-pool-ingress + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-agent} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: hermes-execution-mediator} + ports: [{protocol: TCP, port: 9007}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-isolation + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator} + policyTypes: [Ingress, Egress] + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + podSelector: + matchLabels: {k8s-app: kube-dns} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: vault} + podSelector: + matchLabels: {app: vault} + ports: [{protocol: TCP, port: 8200}] + - to: + - podSelector: + matchLabels: {app: hermes-agent} + ports: [{protocol: TCP, port: 9007}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: hermes-scm} + podSelector: + matchLabels: {app: hermes-scm-broker} + ports: [{protocol: TCP, port: 9081}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-0 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-1 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-2 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-switchyard-ingress + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-switchyard} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: hermes-execution-worker} + ports: [{protocol: TCP, port: 9005}] diff --git a/services/hermes/execution-worker-rbac.yaml b/services/hermes/execution-worker-rbac.yaml new file mode 100644 index 00000000..89ef915e --- /dev/null +++ b/services/hermes/execution-worker-rbac.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes +automountServiceAccountToken: false +# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded +# token is mounted only into Vault-facing containers and grants no Kubernetes +# API verbs to the model-facing execution worker. diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml new file mode 100644 index 00000000..6794f4da --- /dev/null +++ b/services/hermes/execution-worker-statefulset.yaml @@ -0,0 +1,264 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app: hermes-execution-worker +spec: + serviceName: hermes-execution-worker + replicas: 3 + podManagementPolicy: Parallel + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-execution-worker + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + template: + metadata: + labels: + app: hermes-execution-worker + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes + annotations: + ai.bstein.dev/role: fenced-execution-only + ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates + ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback + ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-worker-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-0: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_0_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-1: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_1_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-2: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_2_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-0: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_0_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-1: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_1_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-2: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_2_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + priorityClassName: scavenger + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/arch, operator: In, values: [arm64]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - {key: node-role.kubernetes.io/accelerator, operator: Exists} + - weight: 50 + preference: + matchExpressions: + - {key: hardware, operator: In, values: [rpi5]} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: hermes-execution-worker + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: hermes-execution-worker + initContainers: + - name: stage-worker-access + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker] + env: + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access} + - name: HERMES_WORKER_ORDINAL + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: provider-access, mountPath: /provider-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 32Mi} + limits: {cpu: 100m, memory: 64Mi} + - name: install-provider-clis + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + tools=/worker-data/tools + mkdir -p "${tools}/bin" + if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then + npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ + @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226 + touch "${tools}/.cli-versions-0.147.0-2.1.226" + fi + test -x "${tools}/bin/codex" + test -x "${tools}/bin/claude" + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: tools, mountPath: /worker-data/tools} + resources: + requests: {cpu: 5m, memory: 64Mi} + limits: {cpu: "1", memory: 1Gi} + containers: + - name: execution-worker + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py] + env: + - {name: HERMES_HOME, value: /worker-data} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HOME, value: /worker-data/home} + - {name: CODEX_HOME, value: /provider-access/codex} + - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude} + - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} + - {name: PYTHONPATH, value: /opt/hermes} + - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} + - name: HERMES_WORKER_ORDINAL + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: HERMES_WORKER_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + startupProbe: + exec: + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"] + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + exec: + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"] + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: worker-data, mountPath: /worker-data} + - {name: tools, mountPath: /worker-data/tools, readOnly: true} + - {name: provider-access, mountPath: /provider-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi} + limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi} + volumes: + - name: worker-data + emptyDir: {sizeLimit: 128Mi} + - name: tools + emptyDir: {sizeLimit: 1Gi} + - name: coordinator + configMap: + name: hermes-execution-pool + defaultMode: 0555 + - name: tmp + emptyDir: {sizeLimit: 2Gi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: + audience: vault + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - {key: ca.crt, path: ca.crt} + - downwardAPI: + items: + - {path: namespace, fieldRef: {fieldPath: metadata.namespace}} + volumeClaimTemplates: + - metadata: + name: workspace + labels: + app: hermes-execution-worker + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 30Gi + - metadata: + name: provider-access + labels: + app: hermes-execution-worker + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 1Gi diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 2c2efe77..6becbaf6 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -41,6 +41,13 @@ resources: - oauth2-proxy.yaml - agent-certificate.yaml - agent-ingress.yaml + - execution-worker-rbac.yaml + - execution-worker-statefulset.yaml + - execution-mediator.yaml + - execution-worker-networkpolicy.yaml + +patches: + - path: execution-coordinator-patch.yaml configMapGenerator: - name: hermes-chat-oauth-templates @@ -55,6 +62,36 @@ configMapGenerator: - OPERATOR-RUNBOOK.md=NOTES.md options: disableNameSuffixHash: true + - name: hermes-execution-pool + namespace: hermes + files: + - cli_lane_board.py=scripts/cli_lane_board.py + - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py + - cli_lane_config.py=scripts/cli_lane_config.py + - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py + - cli_lane_evidence.py=scripts/cli_lane_evidence.py + - cli_lane_execution.py=scripts/cli_lane_execution.py + - cli_lane_files.py=scripts/cli_lane_files.py + - cli_lane_finalization.py=scripts/cli_lane_finalization.py + - cli_lane_goal.py=scripts/cli_lane_goal.py + - cli_lane_prompt.py=scripts/cli_lane_prompt.py + - cli_lane_provider.py=scripts/cli_lane_provider.py + - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py + - cli_lane_records.py=scripts/cli_lane_records.py + - cli_lane_recovery.py=scripts/cli_lane_recovery.py + - cli_lane_retention.py=scripts/cli_lane_retention.py + - cli_lane_routing.py=scripts/cli_lane_routing.py + - cli_lane_runner.py=scripts/cli_lane_runner.py + - execution_pool_protocol.py=scripts/execution_pool_protocol.py + - execution_pool_project.py=scripts/execution_pool_project.py + - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py + - execution_pool_server.py=scripts/execution_pool_server.py + - execution_pool_client.py=scripts/execution_pool_client.py + - execution_pool_worker.py=scripts/execution_pool_worker.py + - execution_pool_scm.py=scripts/execution_pool_scm.py + - gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py + - scm_broker_client.py=scm-common/scripts/scm_broker_client.py + - stage_runtime_access.py=scripts/stage_runtime_access.py - name: hermes-coordinator namespace: hermes files: diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml index 17dc1da6..f2d222b5 100644 --- a/services/hermes/scm-common/kustomization.yaml +++ b/services/hermes/scm-common/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization configMapGenerator: - - name: hermes-scm-boundary + - name: hermes-scm-boundary-v2 files: - gitea_api.py=scripts/gitea_api.py - gitea_api_policy.py=scripts/gitea_api_policy.py diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py index a0c4043c..75d6d508 100644 --- a/services/hermes/scm-common/scripts/scm_broker.py +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -7,7 +7,6 @@ import argparse import base64 import json import re -import socket import tempfile import time import urllib.error @@ -45,7 +44,8 @@ GIT_PATH_RE = re.compile( r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" ) FEATURE_REF_RE = re.compile( - r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" + r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|handoff)/" + r"[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" ) @@ -96,7 +96,7 @@ def _spool_bounded( """Copy a fixed-length exchange through a bounded-memory disk spool.""" if not 0 <= length <= maximum: raise PolicyError("SCM request exceeds the safe size limit") - spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns lifetime max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" ) remaining = length @@ -228,7 +228,7 @@ def _receive_prefix(body: bytes | BinaryIO) -> bytes: def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: - """Permit only creation of new, namespaced feature branches.""" + """Permit only creation of new branches in reviewed namespaces.""" body = _receive_prefix(body) _reject_credential_bytes(body, token, "Git request") position = 0 @@ -247,13 +247,11 @@ def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: command = body[position + 4 : position + size].rstrip(b"\n") command = command.split(b"\x00", 1)[0] fields = command.split(b" ") - if len(fields) != 3 or not all( - re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] - ): + if len(fields) != 3 or not all(re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]): raise PolicyError("Git receive-pack ref command is invalid") old_sha, new_sha, raw_ref = fields if old_sha != ZERO_SHA or new_sha == ZERO_SHA: - raise PolicyError("Git broker permits only new feature-branch creation") + raise PolicyError("Git broker permits only new namespaced branch creation") try: ref = raw_ref.decode("ascii") except UnicodeDecodeError as exc: @@ -333,9 +331,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): if len(self.headers.get_all("Content-Length", [])) > 1: raise PolicyError("SCM request has duplicate Content-Length") - def _stream( - self, status: int, content_type: str, body: BinaryIO, length: int - ) -> None: + def _stream(self, status: int, content_type: str, body: BinaryIO, length: int) -> None: self.send_response(status) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(length)) @@ -391,13 +387,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): self._stream(200, expected, body, length) finally: body.close() - except ( - OSError, - PolicyError, - socket.timeout, - urllib.error.URLError, - ValueError, - ): + except (TimeoutError, OSError, PolicyError, urllib.error.URLError, ValueError): self._reject() def do_POST(self) -> None: @@ -409,9 +399,9 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): else: self._git_rpc() except ( + TimeoutError, OSError, PolicyError, - socket.timeout, urllib.error.URLError, ValueError, json.JSONDecodeError, @@ -427,9 +417,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): result = read(data["path"], token=token) else: expected = {"base", "body", "head", "head_sha", "repo", "title"} - if set(data) != expected or not all( - isinstance(data[key], str) for key in expected - ): + if set(data) != expected or not all(isinstance(data[key], str) for key in expected): raise PolicyError("draft request fields are invalid") result = create_draft(token=token, **data) # type: ignore[arg-type] if token.encode("utf-8") in result: @@ -438,15 +426,10 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): def _git_rpc(self) -> None: repo, operation, service = _git_target(self.path) - if ( - operation not in {"git-upload-pack", "git-receive-pack"} - or service != operation - ): + if operation not in {"git-upload-pack", "git-receive-pack"} or service != operation: raise PolicyError("Git RPC operation is outside the allowlist") expected_request = f"application/x-{service}-request" - if self.headers.get_content_type() != expected_request or self.headers.get( - "Transfer-Encoding" - ): + if self.headers.get_content_type() != expected_request or self.headers.get("Transfer-Encoding"): raise PolicyError("Git RPC request type is invalid") length = _content_length(self.headers, MAX_GIT_REQUEST) token = read_token() diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 8a740dbf..0a059911 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -7,6 +7,7 @@ import concurrent.futures import os import sys import time +from collections.abc import Callable from typing import Any from cli_lane_board import _external, _record_board_access_error, _task_value @@ -24,6 +25,16 @@ from cli_lane_recovery import _has_pending_finalization, recover_pending_finaliz from cli_lane_retention import maybe_gc_lane_artifacts +OWNED_WORKSPACES_ONLY = os.environ.get( + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", "" +).strip().lower() in {"1", "true", "yes", "on"} + + +def _owns_local_workspace(task: Any) -> bool: + """Keep legacy/dirty worktree tasks on their existing single-host owner.""" + return bool(str(_task_value(task, "workspace_path", "") or "").strip()) + + def _board_slug(board: Any) -> str: if isinstance(board, dict): return str(board.get("slug") or board.get("id") or "") @@ -59,7 +70,11 @@ def recover_orphans() -> None: continue try: for task in kanban_db.list_tasks(conn): - if _external(task) and str(_task_value(task, "status", "")) == "running": + if ( + _external(task) + and str(_task_value(task, "status", "")) == "running" + and (not OWNED_WORKSPACES_ONLY or _owns_local_workspace(task)) + ): task_id = str(_task_value(task, "id")) run_id = _task_value(task, "current_run_id", None) if not isinstance(run_id, int): @@ -78,7 +93,11 @@ def recover_orphans() -> None: finally: conn.close() -def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: +def claim_ready( + active: set[tuple[str, str]], + limit: int, + eligible: Callable[[str, Any], bool] | None = None, +) -> list[tuple[str, str]]: """Atomically claim external ready tasks across all non-archived boards.""" from hermes_cli import kanban_db @@ -113,6 +132,8 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str or (board, task_id) in active or not assignee.startswith(EXTERNAL_PREFIX) or str(_task_value(task, "status", "")) != "ready" + or (OWNED_WORKSPACES_ONLY and not _owns_local_workspace(task)) + or (eligible is not None and not eligible(board, task)) ): continue try: diff --git a/services/hermes/scripts/cli_lane_evidence.py b/services/hermes/scripts/cli_lane_evidence.py index d85abb0e..529c1fd7 100644 --- a/services/hermes/scripts/cli_lane_evidence.py +++ b/services/hermes/scripts/cli_lane_evidence.py @@ -56,8 +56,12 @@ def _retire_terminal_entry( board_descriptor: int, quarantine_descriptor: int, authority_name: str | None = None, + source_descriptor: int | None = None, ) -> str: """Retire only the inode previously inspected; preserve any replacement.""" + pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat + if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino: + return "replacement" try: current = os.stat( path.name, @@ -66,7 +70,7 @@ def _retire_terminal_entry( ) except FileNotFoundError: return "missing" - if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino: + if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino: return "replacement" # A replacement can itself be staged repeatedly while recovery races a # writer. Keep every generation bound to the original canonical pending @@ -76,7 +80,7 @@ def _retire_terminal_entry( (authority_name or path.name).encode("utf-8") ).hexdigest()[:16] source_digest = hashlib.sha256( - f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode() + f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode() ).hexdigest()[:16] for sequence in range(32): staging = f".retire.{path_digest}.{source_digest}.{sequence}" @@ -97,7 +101,7 @@ def _retire_terminal_entry( dir_fd=quarantine_descriptor, follow_symlinks=False, ) - if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino: + if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino: os.unlink(staging, dir_fd=quarantine_descriptor) os.fsync(quarantine_descriptor) os.fsync(board_descriptor) @@ -132,6 +136,7 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: temporary = f".{path.name}.{uuid.uuid4().hex}.tmp" temporary_stat = None descriptor = None + cleanup_descriptor = None try: descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory) temporary_stat = os.fstat(descriptor) @@ -143,6 +148,7 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: written = os.fstat(stream.fileno()) if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1: raise OSError("terminal evidence is not private and singly linked") + cleanup_descriptor = os.dup(stream.fileno()) try: _rename_noreplace( temporary, @@ -164,7 +170,10 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: temporary_stat, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=cleanup_descriptor, ) + if cleanup_descriptor is not None: + os.close(cleanup_descriptor) os.close(directory) def _terminal_document_digest(document: dict[str, Any]) -> str: diff --git a/services/hermes/scripts/cli_lane_execution.py b/services/hermes/scripts/cli_lane_execution.py index 46a3d7b0..59344a4d 100644 --- a/services/hermes/scripts/cli_lane_execution.py +++ b/services/hermes/scripts/cli_lane_execution.py @@ -30,6 +30,12 @@ from cli_lane_recovery import _has_pending_finalization from cli_lane_routing import fresh_unavailable_provider, select_route +def _goal_rejections(state: dict[str, object]) -> list[object]: + """Return only bounded list-shaped prior judge evidence.""" + history = state.get("goal_rejections", []) + return history if isinstance(history, list) else [] + + def execute_claim(board: str, task_id: str) -> None: """Execute one already-claimed task and commit its outcome to Kanban.""" from hermes_cli import kanban_db @@ -260,16 +266,12 @@ def execute_claim(board: str, task_id: str) -> None: and goal_mode ): heartbeat("local goal-completion judge active") - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] - judge_context = context - if goal_turn > 1 or rejection_history: - judge_context += ( - "\n\nAuthoritative Hermes goal-controller evidence: " - f"current turn {goal_turn}/{goal_max_turns}; prior rejected " - f"reports: {json.dumps(rejection_history[-5:])}." - ) + rejection_history = _goal_rejections(state) + judge_context = context + ( + "\n\nAuthoritative Hermes goal-controller evidence: " + f"current turn {goal_turn}/{goal_max_turns}; prior rejected " + f"reports: {json.dumps(rejection_history[-5:])}." + ) accepted, judge_reason = cli_lane_goal.judge_goal_completion( judge_context, structured, @@ -338,9 +340,7 @@ def execute_claim(board: str, task_id: str) -> None: and deadline - time.monotonic() > 30 ) if can_continue: - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] + rejection_history = _goal_rejections(state) state["goal_rejections"] = [ *rejection_history[-4:], completion_problem, diff --git a/services/hermes/scripts/cli_lane_finalization.py b/services/hermes/scripts/cli_lane_finalization.py index ef39ffc5..b5468296 100644 --- a/services/hermes/scripts/cli_lane_finalization.py +++ b/services/hermes/scripts/cli_lane_finalization.py @@ -89,6 +89,7 @@ def _retire_snapshot( board_descriptor=snapshot.directory_descriptor, quarantine_descriptor=snapshot.directory_descriptor, authority_name=authority_name, + source_descriptor=snapshot.descriptor, ) def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str: diff --git a/services/hermes/scripts/cli_lane_quarantine.py b/services/hermes/scripts/cli_lane_quarantine.py index 8c599184..8de6d3ea 100644 --- a/services/hermes/scripts/cli_lane_quarantine.py +++ b/services/hermes/scripts/cli_lane_quarantine.py @@ -180,6 +180,7 @@ def _quarantine_terminal( source_stat, board_descriptor=board_descriptor, quarantine_descriptor=quarantine_descriptor, + source_descriptor=source_descriptor, ) except OSError as error: # A bad quarantine target must not make an attacker-controlled pending @@ -196,6 +197,7 @@ def _quarantine_terminal( if quarantine_descriptor is not None else board_descriptor ), + source_descriptor=source_descriptor, ) except OSError: retirement = "deferred" diff --git a/services/hermes/scripts/cli_lane_retention.py b/services/hermes/scripts/cli_lane_retention.py index c88df653..9d1b9245 100644 --- a/services/hermes/scripts/cli_lane_retention.py +++ b/services/hermes/scripts/cli_lane_retention.py @@ -50,18 +50,27 @@ def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool: directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) directory_flags |= getattr(os, "O_NOFOLLOW", 0) descriptor = None + source_descriptor = None try: descriptor = os.open(path.parent, directory_flags) + file_flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0) + source_descriptor = os.open(path.name, file_flags, dir_fd=descriptor) + pinned = os.fstat(source_descriptor) + if pinned.st_dev != observed.st_dev or pinned.st_ino != observed.st_ino: + return False outcome = _retire_terminal_entry( path, observed, board_descriptor=descriptor, quarantine_descriptor=descriptor, + source_descriptor=source_descriptor, ) return outcome == "retired" except OSError: return False finally: + if source_descriptor is not None: + os.close(source_descriptor) if descriptor is not None: os.close(descriptor) diff --git a/services/hermes/scripts/execution_pool_client.py b/services/hermes/scripts/execution_pool_client.py new file mode 100644 index 00000000..a1dc6622 --- /dev/null +++ b/services/hermes/scripts/execution_pool_client.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Ordinal-local signing and SCM mediation boundary for a model worker.""" + +from __future__ import annotations + +import json +import os +import threading +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +import cli_lane_goal +from execution_pool_protocol import ( + MAX_WIRE_BYTES, + PROTOCOL_VERSION, + BoundedHTTPServer, + ProtocolError, + canonical_json, + parse_wire, + read_key, + sign_envelope, + verify_envelope, +) +from execution_pool_scm import Boundary as SCMBoundary + + +KEY_PATH = Path( + os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key") +) +COORDINATOR = os.environ.get( + "HERMES_EXECUTION_POOL_URL", + "http://hermes-execution-pool.hermes.svc.cluster.local:9007", +).rstrip("/") +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009")) +RESULT_FIELDS = frozenset( + {"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"} +) + + +def _binding(value: dict[str, Any]) -> dict[str, Any]: + return { + name: value[name] + for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt") + } + + +def _validate_result(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ProtocolError("terminal result payload must be an object") + structured = payload.get("structured") + if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS: + raise ProtocolError("terminal result fields do not match the reviewed schema") + if structured.get("status") not in cli_lane_goal.RESULT_STATUSES: + raise ProtocolError("terminal result status is invalid") + if not isinstance(structured.get("summary"), str) or not structured["summary"].strip(): + raise ProtocolError("terminal result summary is required") + for name in RESULT_FIELDS - {"status", "summary"}: + value = structured.get(name) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ProtocolError(f"terminal result {name} must be a text list") + return payload + + +class ClientBoundary: + """Keep HMAC and SCM authority outside the model-facing container.""" + + def __init__(self, key: bytes, scm: SCMBoundary | None = None): + self.key = key + self.scm = scm or SCMBoundary(key) + self.current: dict[str, Any] | None = None + self.lock = threading.RLock() + + def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]: + request = urllib.request.Request( + COORDINATOR + path, + data=canonical_json(envelope), + method="POST", + headers={"Content-Type": "application/json", "Cache-Control": "no-store"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read(MAX_WIRE_BYTES + 1) + if len(body) > MAX_WIRE_BYTES: + raise ProtocolError("coordinator response exceeds the wire limit") + return verify_envelope(self.key, json.loads(body)) + + def poll(self) -> dict[str, Any]: + poll_binding = { + "board": "", + "task_id": "", + "run_id": "", + "worker_ordinal": ORDINAL, + "attempt": 0, + } + response = self._post( + "/v1/poll", + sign_envelope(self.key, "poll", poll_binding, {"ready": True}), + ) + with self.lock: + if response["kind"] == "ack": + self.current = None + return {"assignment": None} + if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL: + raise ProtocolError("coordinator returned a foreign assignment") + checkout = self.scm.checkout(response) + self.current = response + assignment = { + **_binding(response), + "payload": response["payload"], + "workspace": checkout["workspace"], + "baseline_sha": checkout["baseline_sha"], + "protocol_version": PROTOCOL_VERSION, + } + return {"assignment": assignment} + + def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if not isinstance(supplied, dict): + raise ProtocolError("local request binding must be an object") + if self.current is None or supplied != _binding(self.current): + raise ProtocolError("local request does not own the current assignment") + return self.current, dict(supplied) + + def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]: + payload = request.get("payload") + if not isinstance(payload, dict): + raise ProtocolError("heartbeat payload must be an object") + with self.lock: + _assignment, binding = self._current_for(request.get("binding")) + response = self._post( + "/v1/heartbeat", + sign_envelope(self.key, "heartbeat", binding, payload), + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") + return {"ack": response["payload"]} + + def finish(self, request: dict[str, Any]) -> dict[str, Any]: + payload = _validate_result(request.get("payload")) + with self.lock: + assignment, binding = self._current_for(request.get("binding")) + structured = payload["structured"] + if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0: + submission = self.scm.submit(assignment, request) + pull = str(submission.get("pull_request") or "") + if pull and pull not in structured["artifacts"]: + structured["artifacts"].append(pull) + response = self._post( + "/v1/result", sign_envelope(self.key, "result", binding, payload) + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") + if response["payload"].get("accepted"): + self.current = None + return {"ack": response["payload"], "structured": structured} + + +def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}" + + def _reply(self, status: int, value: dict[str, Any]) -> None: + body = canonical_json(value) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + value = {"ready": True, "protocol_version": PROTOCOL_VERSION} + self._reply(200, value) if self.path == "/ready" else self._reply( + 404, {"error": "not found"} + ) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("Content-Length", "0")) + request = ( + parse_wire(self.rfile.read(length)) + if 0 < length <= MAX_WIRE_BYTES + else None + ) + allowed = {"operation", "binding", "payload", "title", "body"} + if not request or set(request) - allowed: + raise ProtocolError("invalid local mediator request") + operation = str(request.get("operation") or "") + routes = { + "poll": boundary.poll, + "heartbeat": lambda: boundary.heartbeat(request), + "finish": lambda: boundary.finish(request), + } + if operation not in routes: + raise ProtocolError("unsupported local mediator operation") + self._reply(200, routes[operation]()) + except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error: + self._reply(409, {"error": str(error)[:2000]}) + + def log_message(self, _format: str, *_arguments: Any) -> None: + return + + return Handler + + +def main() -> int: + if ORDINAL not in range(3): + raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2") + key = read_key(KEY_PATH) + BoundedHTTPServer( + ("0.0.0.0", PORT), + handler_factory(ClientBoundary(key)), + max_workers=4, + ).serve_forever() + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_coordinator.py b/services/hermes/scripts/execution_pool_coordinator.py new file mode 100644 index 00000000..6fc0f5a4 --- /dev/null +++ b/services/hermes/scripts/execution_pool_coordinator.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Coordinator-only Kanban claim/finalize bridge for three Hermes workers.""" + +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +import cli_lane_goal +import cli_lane_dispatch +from cli_lane_config import canonical_run_id +from execution_pool_project import ( + ProjectPolicyError, + distributed_workspace_eligible, + resolve_assignment, +) +from execution_pool_protocol import ( + MAX_ACTIVITY_BYTES, + PoolStore, + ProtocolError, + derive_ordinal_key, + envelope_ordinal, + sign_envelope, + verify_envelope, +) + +REDACTIONS = ( + re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"), + re.compile( + r"(?i)(authorization|token|secret|password|api[_-]?key|" + r"refresh[_-]?token|access[_-]?token)[\"']?\s*[:=]\s*[\"']?[^\s\"',;}]+" + ), + re.compile(r"\b(?:sk-ant-|sk-proj-|rt_)[A-Za-z0-9_-]{12,}\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), + re.compile(r"(?i)https://[^/@\s]+@scm\.bstein\.dev"), +) + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]: + """Resolve SCM only through the canonical board/project registry.""" + return resolve_assignment(board, task) + + +def assignment_payload( + kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac" +) -> dict[str, Any]: + context = kanban_db.build_worker_context(connection, str(_task_value(task, "id"))) + if not isinstance(context, str): + context = json.dumps(context, default=str, sort_keys=True) + encoded = context.encode("utf-8") + if len(encoded) > 32 * 1024: + raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit") + repo_url, branch, base_branch = resolve_scm(task, board) + runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60) + runtime = max(60, min(runtime, 12 * 60 * 60)) + return { + "context": context, + "assignee": str(_task_value(task, "assignee", "cli-auto") or "cli-auto"), + "repo_url": repo_url, + "branch": branch, + "base_branch": base_branch, + "max_runtime_seconds": runtime, + "deadline_unix": int(time.time()) + runtime, + "goal_mode": bool(_task_value(task, "goal_mode", False)), + "goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)), + } + + +def sanitize_activity(value: Any) -> str: + """Bound and redact worker output before it reaches the shared UI log.""" + text = str(value or "").replace("\x00", "") + text = "".join(character for character in text if character in "\n\t" or ord(character) >= 32) + for pattern in REDACTIONS: + text = pattern.sub("[REDACTED]", text) + return text.encode("utf-8")[:MAX_ACTIVITY_BYTES].decode("utf-8", "ignore") + + +def _append_activity(kanban_db: Any, envelope: dict[str, Any]) -> None: + payload = envelope["payload"] + if not isinstance(payload, dict): + raise ProtocolError("heartbeat payload must be an object") + activity = sanitize_activity(payload.get("activity")) + if not activity: + return + path = Path(kanban_db.worker_log_path(envelope["task_id"], board=envelope["board"])) + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + raise ProtocolError("Kanban activity log must not be a symlink") + flags = os.O_APPEND | os.O_CREAT | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + os.write(descriptor, activity.encode("utf-8")) + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _append_terminal_activity(kanban_db: Any, record: dict[str, Any]) -> None: + """Append one terminal batch exactly once across result retries/restarts.""" + payload = record.get("result") + activity = payload.get("final_activity") if isinstance(payload, dict) else "" + activity = sanitize_activity(activity) + if not activity: + return + marker = f"\n[execution-pool-result:{record['result_digest']}]\n" + path = Path(kanban_db.worker_log_path(record["task_id"], board=record["board"])) + if path.is_symlink(): + raise ProtocolError("Kanban activity log must not be a symlink") + if path.exists(): + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + size = os.fstat(descriptor).st_size + os.lseek(descriptor, max(0, size - 64 * 1024), os.SEEK_SET) + if marker.encode() in os.read(descriptor, 64 * 1024): + return + finally: + os.close(descriptor) + _append_activity( + kanban_db, + {**record, "payload": {"activity": marker + activity}}, + ) + + +class Coordinator: + """Own all transitions between durable assignments and Hermes Kanban.""" + + def __init__(self, key: bytes, store: PoolStore): + self.master_key = key + self.store = store + self._kanban_lock = threading.RLock() + + def _key(self, envelope: dict[str, Any]) -> bytes: + return derive_ordinal_key(self.master_key, envelope_ordinal(envelope)) + + def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]: + return verify_envelope(self._key(envelope), envelope, expected_kind=kind) + + def _sign( + self, kind: str, binding: dict[str, Any], payload: dict[str, Any] + ) -> dict[str, Any]: + key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"])) + return sign_envelope(key, kind, binding, payload) + + @staticmethod + def _binding(record: dict[str, Any]) -> dict[str, Any]: + return {name: record[name] for name in ( + "board", "task_id", "run_id", "worker_ordinal", "attempt" + )} + + def poll(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "poll") + record = self.store.offer(int(verified["worker_ordinal"])) + if record is None: + binding = {**self._binding({ + "board": "", "task_id": "", "run_id": "", + "worker_ordinal": verified["worker_ordinal"], "attempt": 0, + })} + return self._sign("ack", binding, {"assignment": None}) + return self._sign("assignment", self._binding(record), record["payload"]) + + def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "heartbeat") + accepted, duplicate = self.store.heartbeat(verified) + from hermes_cli import kanban_db + + payload = verified["payload"] + note = sanitize_activity(payload.get("note") if isinstance(payload, dict) else "")[:1000] + with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]): + connection = kanban_db.connect(board=verified["board"]) + try: + run_id = canonical_run_id(verified["run_id"]) + if run_id is None: + raise ProtocolError("worker run ID is not a canonical SQLite run") + alive = kanban_db.heartbeat_worker( + connection, verified["task_id"], + note=note or "distributed worker active", + expected_run_id=run_id, + ) + if not alive: + raise ProtocolError("Kanban run no longer owns this worker") + if not duplicate: + _append_activity(kanban_db, verified) + finally: + connection.close() + return self._sign( + "ack", self._binding(verified), + {"accepted": accepted, "duplicate": duplicate}, + ) + + def result(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "result") + record, duplicate = self.store.accept_result(verified) + if record.get("state") == "result": + self.finalize(record) + return self._sign( + "ack", self._binding(verified), + {"accepted": True, "duplicate": duplicate}, + ) + + def finalize(self, record: dict[str, Any]) -> None: + """Apply a result only if this exact run remains authoritative.""" + from hermes_cli import kanban_db + + binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + return + payload = record.get("result") + if not isinstance(payload, dict): + raise ProtocolError("result payload must be an object") + structured = payload.get("structured") + if not isinstance(structured, dict): + structured = {} + with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + task = kanban_db.get_task(connection, binding["task_id"]) + current_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) if task is not None else None + if current_run_id is None or current_run_id != run_id: + self.store.finalize(binding, "stale") + return + _append_terminal_activity(kanban_db, record) + metadata = { + "executor": "distributed-execution-pool", + "worker_ordinal": binding["worker_ordinal"], + "attempt": binding["attempt"], + "node": str(payload.get("node") or "unknown")[:128], + "route": payload.get("route") if isinstance(payload.get("route"), dict) else {}, + "provider_sessions": payload.get("provider_sessions") + if isinstance(payload.get("provider_sessions"), dict) else {}, + "changed_files": structured.get("changed_files", []), + "tests_run": structured.get("tests_run", []), + "artifacts": structured.get("artifacts", []), + "findings": structured.get("findings", []), + "blockers": structured.get("blockers", []), + } + problem = cli_lane_goal.unfinished_result_reason(structured) + if ( + structured.get("status") == "completed" + and int(payload.get("returncode", 1)) == 0 + and problem is None + ): + changed = kanban_db.complete_task( + connection, binding["task_id"], + result=json.dumps(structured, sort_keys=True), + summary=str(structured.get("summary") or "Completed"), + metadata=metadata, expected_run_id=run_id, + ) + else: + reason = problem or "; ".join(map(str, structured.get("blockers", []))) + reason = reason or str(structured.get("summary") or "worker failed") + changed = kanban_db.block_task( + connection, binding["task_id"], reason=reason, + kind="transient" if payload.get("capacity_failure") else "capability", + expected_run_id=run_id, + ) + self.store.finalize(binding, "finalized" if changed else "stale") + finally: + connection.close() + + def recover_results(self) -> None: + for record in self.store.pending_results(): + try: + self.finalize(record) + except (OSError, sqlite3.Error) as error: + print(f"result recovery deferred: {error}", file=sys.stderr, flush=True) + + def expire_leases(self) -> None: + """Fence dead attempts; after bounded retries, surface and release the run.""" + from hermes_cli import kanban_db + + for record in self.store.expire_leases(): + if record.get("state") != "lease_failed": + continue + binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + continue + with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + changed = kanban_db.block_task( + connection, + binding["task_id"], + reason=( + "Distributed worker lease expired after " + f"{binding['attempt']} fenced attempts" + ), + kind="transient", + expected_run_id=run_id, + ) + self.store.finalize( + binding, "finalized" if changed else "stale" + ) + finally: + connection.close() + + def reconcile(self) -> None: + """Recover the narrow claim/assignment crash gaps without double execution.""" + from hermes_cli import kanban_db + + active = self.store.active_assignments() + active_runs = { + (record["board"], record["task_id"], record["run_id"]) + for record in active + } + for record in active: + try: + with kanban_db.scoped_current_board(record["board"]): + connection = kanban_db.connect(board=record["board"]) + try: + task = kanban_db.get_task(connection, record["task_id"]) + current_run_id = ( + canonical_run_id(_task_value(task, "current_run_id", None)) + if task + else None + ) + current = str(current_run_id or "") + status = str(_task_value(task, "status", "") or "") if task else "" + finally: + connection.close() + if current != record["run_id"] or status != "running": + self.store.finalize(record, "stale") + except (OSError, sqlite3.Error): + continue + ordinals = self.store.available_ordinals() + if not ordinals: + return + for raw_board in kanban_db.list_boards(include_archived=False): + board = cli_lane_dispatch._board_slug(raw_board) + if not board: + continue + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + tasks = kanban_db.list_tasks(connection) + for task in tasks: + task_id = str(_task_value(task, "id", "") or "") + raw_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + run_id = str(raw_run_id or "") + assignee = str(_task_value(task, "assignee", "") or "") + if ( + not ordinals + or not task_id + or not run_id + or str(_task_value(task, "status", "")) != "running" + or not assignee.startswith("cli-") + or not distributed_workspace_eligible(task) + or (board, task_id, run_id) in active_runs + ): + continue + try: + payload = assignment_payload(kanban_db, connection, task, board) + except Exception as error: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed assignment recovery failed: " + f"{type(error).__name__}: {error}" + ), + kind="capability", expected_run_id=raw_run_id, + ) + continue + ordinal = ordinals.pop(0) + self.store.add( + { + "board": board, "task_id": task_id, + "run_id": run_id, "worker_ordinal": ordinal, + "attempt": 1, + }, + payload, + ) + finally: + connection.close() + + def dispatch(self) -> None: + """Claim up to the free ordinal count and materialize assignments.""" + from hermes_cli import kanban_db + + ordinals = self.store.available_ordinals() + if not ordinals: + return + claimed = cli_lane_dispatch.claim_ready( + set(), len(ordinals), + lambda _board, task: distributed_workspace_eligible(task), + ) + for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False): + run_id = "" + database_run_id: int | None = None + try: + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + task = kanban_db.get_task(connection, task_id) + if task is None: + continue + if not distributed_workspace_eligible(task): + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is not None: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed claim fenced because an existing " + "workspace is owned by the local lane" + ), + kind="capability", + expected_run_id=raw_run, + ) + continue + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is None: + raise ProjectPolicyError("claimed task has no canonical run ID") + database_run_id = raw_run + run_id = str(raw_run) + payload = assignment_payload(kanban_db, connection, task, board) + finally: + connection.close() + binding = { + "board": board, "task_id": task_id, "run_id": run_id, + "worker_ordinal": ordinal, "attempt": 1, + } + self.store.add(binding, payload) + except Exception as error: + reason = f"Distributed assignment preparation failed: {type(error).__name__}: {error}" + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + kanban_db.block_task( + connection, task_id, reason=reason, kind="capability", + expected_run_id=database_run_id, + ) + finally: + connection.close() + + +def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]: + """Compatibility export for tests and the mounted coordinator entry point.""" + from execution_pool_server import handler_factory as factory + + return factory(coordinator) + + +def main() -> int: + from execution_pool_server import run + + return run(Coordinator) + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_project.py b/services/hermes/scripts/execution_pool_project.py new file mode 100644 index 00000000..d4b6f382 --- /dev/null +++ b/services/hermes/scripts/execution_pool_project.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Canonical Atlas project and Git-ref policy for distributed execution.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from pathlib import Path +from typing import Any + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +PROJECT_ROOT = DATA_ROOT / "workspace/projects" +BOARD_ROOT = DATA_ROOT / "kanban/boards" +ATLAS_REPO = re.compile( + r"https://scm\.bstein\.dev/atlas/(?P[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z" +) +SAFE_PREFIXES = frozenset( + {"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"} +) +IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") +GIT = "/usr/bin/git" +MAX_BOARD_BYTES = 64 * 1024 + + +class ProjectPolicyError(ValueError): + """Canonical project metadata or a requested ref failed closed.""" + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def _run_git(workdir: Path, *arguments: str) -> str: + completed = subprocess.run( + [GIT, "-C", str(workdir), *arguments], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + env={ + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }, + ) + if completed.returncode: + raise ProjectPolicyError("canonical project Git metadata is unavailable") + return completed.stdout.strip() + + +def validate_branch(value: object, *, feature: bool) -> str: + """Validate a complete local branch name with Git and a reviewed namespace.""" + if not isinstance(value, str) or not value or len(value) > 200: + raise ProjectPolicyError("branch name exceeds the safe limit") + if not value.isascii() or value.startswith("-"): + raise ProjectPolicyError("branch name must be canonical ASCII") + prefix = value.split("/", 1)[0] + if feature and ("/" not in value or prefix not in SAFE_PREFIXES): + raise ProjectPolicyError("task branch is outside the reviewed namespace") + completed = subprocess.run( + [GIT, "check-ref-format", f"refs/heads/{value}"], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + env={"PATH": "/usr/bin:/bin", "GIT_CONFIG_NOSYSTEM": "1"}, + ) + if completed.returncode: + raise ProjectPolicyError("branch name is not a valid Git ref") + return value + + +def _read_board(board: str) -> dict[str, Any]: + if not IDENTIFIER.fullmatch(board): + raise ProjectPolicyError("board slug is invalid") + path = BOARD_ROOT / board / "board.json" + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BOARD_BYTES: + raise ProjectPolicyError("board registry entry is not a bounded regular file") + raw = os.read(descriptor, MAX_BOARD_BYTES + 1) + finally: + os.close(descriptor) + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError) as error: + raise ProjectPolicyError("board registry entry is malformed") from error + if not isinstance(value, dict) or value.get("slug") != board or value.get("archived") is True: + raise ProjectPolicyError("board registry identity is unavailable or archived") + return value + + +def resolve_project(board: str) -> tuple[str, str, Path]: + """Resolve repo and base exclusively from the canonical board registry. + + A board may be created before its primary checkout. In that case the + Atlas repository naming contract and ``main`` are the registry defaults; + once a checkout exists, its credential-free origin and remote HEAD must + agree with that identity. This keeps a missing checkout from silently + routing work to some other project's repository. + """ + entry = _read_board(board) + raw_workdir = entry.get("default_workdir") + if not isinstance(raw_workdir, str) or not Path(raw_workdir).is_absolute(): + raise ProjectPolicyError("board default_workdir is missing") + workdir = Path(raw_workdir).resolve(strict=False) + project_root = PROJECT_ROOT.resolve(strict=True) + try: + workdir.relative_to(project_root) + except ValueError as error: + raise ProjectPolicyError("board workdir is outside the Atlas project registry") from error + remote = f"https://scm.bstein.dev/atlas/{board}.git" + if not ATLAS_REPO.fullmatch(remote): + raise ProjectPolicyError("board repository identity is invalid") + if not workdir.exists(): + return remote, "main", workdir + if not workdir.is_dir(): + raise ProjectPolicyError("board checkout is not a directory") + checkout_remote = _run_git(workdir, "remote", "get-url", "origin") + if "@" in checkout_remote or checkout_remote != remote: + raise ProjectPolicyError("board origin disagrees with the Atlas registry") + try: + base = _run_git(workdir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") + if not base.startswith("origin/"): + raise ProjectPolicyError("origin HEAD is not canonical") + base = base.removeprefix("origin/") + except ProjectPolicyError: + base = "main" + return remote, validate_branch(base, feature=False), workdir + + +def resolve_assignment(board: str, task: Any) -> tuple[str, str, str]: + """Resolve the exact repo/base and safe task branch for one board task.""" + repo, base, _workdir = resolve_project(board) + task_id = str(_task_value(task, "id", "") or "") + if not IDENTIFIER.fullmatch(task_id): + raise ProjectPolicyError("task identity is invalid") + branch = str(_task_value(task, "branch_name", "") or f"wt/{task_id}") + return repo, validate_branch(branch, feature=True), base + + +def distributed_workspace_eligible(task: Any) -> bool: + """Only pathless tasks migrate; an existing worktree stays with its local owner.""" + return not str(_task_value(task, "workspace_path", "") or "").strip() diff --git a/services/hermes/scripts/execution_pool_protocol.py b/services/hermes/scripts/execution_pool_protocol.py new file mode 100644 index 00000000..599279be --- /dev/null +++ b/services/hermes/scripts/execution_pool_protocol.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Authenticated, bounded, restart-safe Hermes execution-pool protocol.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import re +import sqlite3 +import stat +import threading +import time +import uuid +from http.server import ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +MAX_WIRE_BYTES = 64 * 1024 +MAX_ACTIVITY_BYTES = 12 * 1024 +MAX_CLOCK_SKEW = 30 +MAX_ENVELOPE_LIFETIME = 300 +PROTOCOL_VERSION = 2 +IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +KINDS = frozenset({"poll", "assignment", "heartbeat", "result", "ack"}) + + +class ProtocolError(ValueError): + """A request failed the authenticated pool contract.""" + + +class BoundedHTTPServer(ThreadingHTTPServer): + """Bound concurrent requests and slow clients on internal pool channels.""" + + daemon_threads = True + request_queue_size = 8 + + def __init__(self, *args: Any, max_workers: int = 8, **kwargs: Any): + self._slots = threading.BoundedSemaphore(max(1, min(max_workers, 16))) + super().__init__(*args, **kwargs) + + def get_request(self) -> tuple[Any, Any]: + request, address = super().get_request() + request.settimeout(15) + return request, address + + def process_request(self, request: Any, client_address: Any) -> None: + if not self._slots.acquire(blocking=False): + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + raise + + def process_request_thread(self, request: Any, client_address: Any) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() + + +def canonical_json(value: Any) -> bytes: + """Encode one value deterministically for digests and signatures.""" + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: + """Durably replace one bounded pool document without following symlinks.""" + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.parent.is_symlink(): + raise ProtocolError("pool state directory must not be a symlink") + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, mode) + try: + remaining = memoryview(json.dumps(value, indent=2, sort_keys=True).encode() + b"\n") + while remaining: + remaining = remaining[os.write(descriptor, remaining) :] + os.fsync(descriptor) + os.fchmod(descriptor, mode) + finally: + os.close(descriptor) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def payload_digest(payload: Any) -> str: + return hashlib.sha256(canonical_json(payload)).hexdigest() + + +def derive_ordinal_key(master: bytes, ordinal: int) -> bytes: + """Derive one cryptographically isolated worker authority from the pool root.""" + if ordinal not in range(3) or not 32 <= len(master) <= 4096: + raise ProtocolError("pool key derivation input is invalid") + context = f"hermes-execution-pool-v2:worker:{ordinal}".encode() + return hmac.new(master, context, hashlib.sha256).hexdigest().encode() + + +def envelope_ordinal(envelope: Any) -> int: + """Read only the bounded ordinal needed to select a verification key.""" + if not isinstance(envelope, dict): + raise ProtocolError("invalid pool message") + value = envelope.get("worker_ordinal") + if type(value) is not int or value not in range(3): + raise ProtocolError("worker binding is outside the pool") + return value + + +def read_key(path: Path) -> bytes: + """Read a private regular file without following a final symlink.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ProtocolError(f"pool key is unavailable: {error}") from error + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077: + raise ProtocolError("pool key must be a private regular file") + value = os.read(descriptor, 4097).strip() + finally: + os.close(descriptor) + if len(value) < 32 or len(value) > 4096: + raise ProtocolError("pool key length is outside the safe range") + return value + + +def _identifier(name: str, value: Any, *, allow_empty: bool = False) -> str: + if not isinstance(value, str): + raise ProtocolError(f"invalid {name}") + text = value + if allow_empty and not text: + return text + if not IDENTIFIER.fullmatch(text): + raise ProtocolError(f"invalid {name}") + return text + + +def sign_envelope( + key: bytes, + kind: str, + binding: dict[str, Any], + payload: Any, + *, + now: int | None = None, + lifetime: int = 120, + delivery_id: str | None = None, +) -> dict[str, Any]: + """Bind a message to the exact Kanban run, ordinal, and attempt.""" + current = int(time.time()) if now is None else int(now) + lifetime = max(1, min(int(lifetime), MAX_ENVELOPE_LIFETIME)) + envelope = { + "version": PROTOCOL_VERSION, + "kind": kind, + "board": str(binding.get("board") or ""), + "task_id": str(binding.get("task_id") or ""), + "run_id": str(binding.get("run_id") or ""), + "worker_ordinal": int(binding.get("worker_ordinal", -1)), + "attempt": int(binding.get("attempt", 0)), + "delivery_id": delivery_id or str(uuid.uuid4()), + "issued_at": current, + "expires_at": current + lifetime, + "payload_digest": payload_digest(payload), + "payload": payload, + } + if kind not in KINDS: + raise ProtocolError("unsupported message kind") + unsigned = canonical_json(envelope) + if len(unsigned) > MAX_WIRE_BYTES: + raise ProtocolError("pool message exceeds the wire limit") + envelope["signature"] = hmac.new(key, unsigned, hashlib.sha256).hexdigest() + return envelope + + +def verify_envelope( + key: bytes, + envelope: Any, + *, + expected_kind: str | None = None, + now: int | None = None, +) -> dict[str, Any]: + """Verify structure, lifetime, digest, and HMAC before using a message.""" + if not isinstance(envelope, dict) or len(canonical_json(envelope)) > MAX_WIRE_BYTES: + raise ProtocolError("invalid or oversized pool message") + required = {"version", "kind", "board", "task_id", "run_id", "worker_ordinal", "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", "payload", "signature"} + if set(envelope) != required or envelope.get("version") != PROTOCOL_VERSION: + raise ProtocolError( + f"pool message fields do not match version {PROTOCOL_VERSION}" + ) + kind = envelope["kind"] + if not isinstance(kind, str): + raise ProtocolError("unexpected message kind") + if kind not in KINDS or (expected_kind and kind != expected_kind): + raise ProtocolError("unexpected message kind") + empty_binding = kind in {"poll", "ack"} + _identifier("board", envelope["board"], allow_empty=empty_binding) + _identifier("task_id", envelope["task_id"], allow_empty=empty_binding) + _identifier("run_id", envelope["run_id"], allow_empty=empty_binding) + _identifier("delivery_id", envelope["delivery_id"]) + numeric = tuple(envelope[name] for name in ("worker_ordinal", "attempt", "issued_at", "expires_at")) + if any(type(value) is not int for value in numeric): + raise ProtocolError("invalid numeric binding") + ordinal, attempt, issued, expires = numeric + if ordinal not in range(3) or attempt < 0: + raise ProtocolError("worker binding is outside the pool") + current = int(time.time()) if now is None else int(now) + if issued > current + MAX_CLOCK_SKEW or expires < current - MAX_CLOCK_SKEW: + raise ProtocolError("pool message is outside its validity window") + if expires <= issued or expires - issued > MAX_ENVELOPE_LIFETIME: + raise ProtocolError("pool message lifetime is invalid") + if envelope["payload_digest"] != payload_digest(envelope["payload"]): + raise ProtocolError("pool payload digest mismatch") + signature = str(envelope["signature"]) + unsigned = dict(envelope) + unsigned.pop("signature") + expected = hmac.new(key, canonical_json(unsigned), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + raise ProtocolError("pool message authentication failed") + return envelope + + +def parse_wire(body: bytes) -> dict[str, Any]: + if not body or len(body) > MAX_WIRE_BYTES: + raise ProtocolError("empty or oversized request") + try: + value = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ProtocolError("malformed JSON request") from error + if not isinstance(value, dict): + raise ProtocolError("request must be a JSON object") + return value + + +class PoolStore: + """Coordinator-owned durable assignments; never stores provider secrets.""" + + def __init__(self, path: Path, lease_seconds: int = 90): + self.path = path + self.lease_seconds = max(60, min(int(lease_seconds), 600)) + self._lock = threading.RLock() + path.parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, timeout=10, isolation_level=None) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA busy_timeout=10000") + return connection + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS assignments ( + board TEXT NOT NULL, task_id TEXT NOT NULL, run_id TEXT NOT NULL, + worker_ordinal INTEGER NOT NULL CHECK(worker_ordinal BETWEEN 0 AND 2), + attempt INTEGER NOT NULL, assignment_digest TEXT NOT NULL, + payload_json TEXT NOT NULL, state TEXT NOT NULL, + lease_until REAL NOT NULL DEFAULT 0, last_heartbeat REAL NOT NULL DEFAULT 0, + result_digest TEXT, result_json TEXT, created_at REAL NOT NULL, + updated_at REAL NOT NULL, PRIMARY KEY(board, task_id, run_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS one_live_assignment_per_worker + ON assignments(worker_ordinal) WHERE state IN ('assigned','running','result'); + CREATE TABLE IF NOT EXISTS deliveries ( + delivery_id TEXT PRIMARY KEY, kind TEXT NOT NULL, digest TEXT NOT NULL, + received_at REAL NOT NULL + ); + """ + ) + + @staticmethod + def _record(row: sqlite3.Row | None) -> dict[str, Any] | None: + if row is None: + return None + value = dict(row) + value["payload"] = json.loads(value.pop("payload_json")) + if value.get("result_json"): + value["result"] = json.loads(value["result_json"]) + return value + + def add(self, binding: dict[str, Any], payload: dict[str, Any]) -> bool: + """Create exactly one assignment for a claimed run and free ordinal.""" + now = time.time() + digest = payload_digest(payload) + values = ( + binding["board"], binding["task_id"], binding["run_id"], + binding["worker_ordinal"], binding["attempt"], digest, + canonical_json(payload).decode(), "assigned", now, now, + ) + with self._lock, self._connect() as connection: + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + """INSERT INTO assignments + (board,task_id,run_id,worker_ordinal,attempt,assignment_digest, + payload_json,state,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + values, + ) + connection.commit() + return True + except sqlite3.IntegrityError as error: + connection.rollback() + existing = connection.execute( + """SELECT assignment_digest,worker_ordinal,attempt FROM assignments + WHERE board=? AND task_id=? AND run_id=?""", + values[:3], + ).fetchone() + if existing and tuple(existing) == ( + digest, binding["worker_ordinal"], binding["attempt"] + ): + return False + if existing: + raise ProtocolError("conflicting duplicate assignment") from error + raise ProtocolError("worker ordinal already has a live assignment") from error + + def available_ordinals(self) -> list[int]: + with self._connect() as connection: + rows = connection.execute( + "SELECT worker_ordinal FROM assignments WHERE state IN ('assigned','running','result')" + ).fetchall() + occupied = {int(row[0]) for row in rows} + return [ordinal for ordinal in range(3) if ordinal not in occupied] + + def active_assignments(self) -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute( + "SELECT * FROM assignments WHERE state IN ('assigned','running','result')" + ).fetchall() + return [self._record(row) or {} for row in rows] + + def offer(self, ordinal: int) -> dict[str, Any] | None: + """Return the ordinal's durable assignment, preserving restart identity.""" + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + """SELECT * FROM assignments WHERE worker_ordinal=? + AND state IN ('assigned','running') ORDER BY created_at LIMIT 1""", + (ordinal,), + ).fetchone() + if row is not None: + connection.execute( + """UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (now + self.lease_seconds, now, now, row["board"], row["task_id"], row["run_id"]), + ) + connection.commit() + return self._record(row) + + def _matching(self, connection: sqlite3.Connection, envelope: dict[str, Any]) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + if row is None: + raise ProtocolError("assignment is unknown or stale") + if int(row["worker_ordinal"]) != int(envelope["worker_ordinal"]): + raise ProtocolError("worker ordinal does not own this assignment") + if int(row["attempt"]) != int(envelope["attempt"]): + raise ProtocolError("assignment attempt is stale") + return row + + def heartbeat(self, envelope: dict[str, Any]) -> tuple[bool, bool]: + now = time.time() + delivery_digest = payload_digest( + { + name: envelope[name] + for name in ( + "kind", "board", "task_id", "run_id", "worker_ordinal", + "attempt", "payload_digest", + ) + } + ) + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + duplicate = connection.execute( + "SELECT digest FROM deliveries WHERE delivery_id=?", + (envelope["delivery_id"],), + ).fetchone() + row = self._matching(connection, envelope) + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment is no longer running") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + if duplicate and duplicate[0] != delivery_digest: + raise ProtocolError("delivery identifier was reused") + if not duplicate: + connection.execute( + "INSERT INTO deliveries VALUES (?,?,?,?)", + (envelope["delivery_id"], "heartbeat", delivery_digest, now), + ) + connection.execute( + "UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?", + (now + self.lease_seconds, now, now, envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + return True, bool(duplicate) + + def accept_result(self, envelope: dict[str, Any]) -> tuple[dict[str, Any], bool]: + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = self._matching(connection, envelope) + digest = envelope["payload_digest"] + if row["result_digest"]: + if row["result_digest"] != digest: + raise ProtocolError("conflicting result for completed delivery") + connection.rollback() + return self._record(row) or {}, True + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment cannot accept a result") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + connection.execute( + """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (digest, canonical_json(envelope["payload"]).decode(), now, + envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + return self._record(row) or {}, False + + def pending_results(self) -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute("SELECT * FROM assignments WHERE state='result' ORDER BY updated_at").fetchall() + return [self._record(row) or {} for row in rows] + + def expire_leases( + self, *, now: float | None = None, max_attempts: int = 3 + ) -> list[dict[str, Any]]: + """Fence expired attempts and re-offer or terminally release their ordinals.""" + current = time.time() if now is None else float(now) + maximum = max(1, min(int(max_attempts), 10)) + changed: list[dict[str, Any]] = [] + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + rows = connection.execute( + """SELECT * FROM assignments WHERE state='running' + AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""", + (current,), + ).fetchall() + for row in rows: + if int(row["attempt"]) >= maximum: + state, attempt = "lease_failed", int(row["attempt"]) + else: + state, attempt = "assigned", int(row["attempt"]) + 1 + connection.execute( + """UPDATE assignments SET state=?,attempt=?,lease_until=0, + last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=? + AND attempt=? AND state='running'""", + ( + state, attempt, current, row["board"], row["task_id"], + row["run_id"], row["attempt"], + ), + ) + updated = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (row["board"], row["task_id"], row["run_id"]), + ).fetchone() + if updated is not None: + changed.append(self._record(updated) or {}) + connection.commit() + return changed + + def finalize(self, binding: dict[str, Any], state: str) -> None: + if state not in {"finalized", "stale"}: + raise ProtocolError("invalid terminal assignment state") + with self._lock, self._connect() as connection: + connection.execute( + "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?", + (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))), + ) + + def garbage_collect(self, retention_seconds: int) -> int: + cutoff = time.time() - max(3600, retention_seconds) + with self._lock, self._connect() as connection: + cursor = connection.execute( + "DELETE FROM assignments WHERE state IN ('finalized','stale') AND updated_at < ?", + (cutoff,), + ) + connection.execute("DELETE FROM deliveries WHERE received_at < ?", (cutoff,)) + return int(cursor.rowcount) diff --git a/services/hermes/scripts/execution_pool_scm.py b/services/hermes/scripts/execution_pool_scm.py new file mode 100644 index 00000000..6badfbb9 --- /dev/null +++ b/services/hermes/scripts/execution_pool_scm.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +import threading +import urllib.parse +from pathlib import Path +from typing import Any + +import scm_broker_client +from execution_pool_project import ATLAS_REPO, validate_branch +from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope + + +WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state")) +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/") +MAX_STATUS_BYTES = 4 * 1024 * 1024 + + +def _git_environment() -> dict[str, str]: + """Run Git without credentials, prompts, ambient config, or hook execution.""" + return { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + + +def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str: + command = [ + "/usr/bin/git", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + *arguments, + ] + completed = subprocess.run( + command, + cwd=cwd, + env=_git_environment(), + stdin=subprocess.DEVNULL, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if completed.returncode: + message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:] + raise RuntimeError(message.strip()) + if len(completed.stdout.encode()) > MAX_STATUS_BYTES: + raise ProtocolError("SCM command output exceeds the safe limit") + return completed.stdout.strip() + + +def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]: + if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL: + raise ProtocolError("assignment does not belong to this worker ordinal") + payload = envelope.get("payload") + if not isinstance(payload, dict): + raise ProtocolError("assignment payload must be an object") + repo = str(payload.get("repo_url") or "") + match = ATLAS_REPO.fullmatch(repo) + if not match: + raise ProtocolError("assignment repository is outside Atlas") + try: + branch = validate_branch(payload.get("branch"), feature=True) + base = validate_branch(payload.get("base_branch"), feature=False) + except ValueError as error: + raise ProtocolError(str(error)) from error + return payload, match.group("repo"), branch, base + + +def _broker_repo(repo: str) -> str: + return f"{BROKER_ORIGIN}/git/atlas/{repo}.git" + + +def workspace_path(envelope: dict[str, Any]) -> Path: + """Derive a private ordinal path; no caller-provided path is accepted.""" + parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id")) + identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") + if any(not identifier.fullmatch(part) for part in parts): + raise ProtocolError("assignment path binding is invalid") + if WORKSPACE_ROOT.is_symlink(): + raise ProtocolError("workspace root must not be a symlink") + workspace_root = WORKSPACE_ROOT.resolve() + root = WORKSPACE_ROOT / "runs" + if root.is_symlink(): + raise ProtocolError("workspace run root must not be a symlink") + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root = root.resolve() + root.relative_to(workspace_root) + candidate = root.joinpath(*parts) + current = root + for part in parts[:-1]: + current /= part + if current.is_symlink(): + raise ProtocolError("workspace parent must not be a symlink") + current.mkdir(mode=0o700, exist_ok=True) + if candidate.is_symlink(): + raise ProtocolError("workspace must not be a symlink") + candidate.resolve(strict=False).relative_to(root) + return candidate + + +def _regular_text(path: Path, maximum: int) -> str: + descriptor = os.open( + path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + ) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size > maximum: + raise ProtocolError("private SCM state is invalid") + raw = os.read(descriptor, maximum + 1) + finally: + os.close(descriptor) + try: + return raw.decode().strip() + except UnicodeError as error: + raise ProtocolError("private SCM state is malformed") from error + + +def _state_path(envelope: dict[str, Any]) -> Path: + if SCM_ROOT.is_symlink(): + raise ProtocolError("private SCM root must not be a symlink") + root = SCM_ROOT.resolve() + root.mkdir(mode=0o700, parents=True, exist_ok=True) + name = "-".join( + str(envelope[name]) for name in ("board", "task_id", "run_id") + ) + path = root / f"{name}.json" + if path.is_symlink(): + raise ProtocolError("private SCM state must not be a symlink") + path.resolve(strict=False).relative_to(root) + return path + + +def _workspace_identity(destination: Path, repo: str, branch: str) -> str: + """Validate the checkout using bounded Git plumbing with all hooks disabled.""" + if destination.is_symlink() or not (destination / ".git").is_dir(): + raise ProtocolError("workspace Git metadata is unavailable") + ref_path = destination / ".git/refs/heads" + for part in branch.split("/"): + ref_path /= part + if ref_path.is_symlink(): + raise ProtocolError("workspace branch ref must not be a symlink") + origin = _run("remote", "get-url", "origin", cwd=destination) + if origin != f"https://scm.bstein.dev/atlas/{repo}.git": + raise ProtocolError("workspace origin does not match assignment") + broker = _run("remote", "get-url", "hermes-broker", cwd=destination) + if broker != _broker_repo(repo): + raise ProtocolError("workspace broker remote does not match assignment") + current = _run("symbolic-ref", "--short", "HEAD", cwd=destination) + if current != branch: + raise ProtocolError("workspace branch does not match assignment") + head = _run("rev-parse", "--verify", "HEAD", cwd=destination) + if not re.fullmatch(r"[0-9a-f]{40,64}", head): + raise ProtocolError("workspace HEAD is invalid") + return head + + +class Boundary: + """The only process allowed to turn model output into an SCM/result handoff.""" + + def __init__(self, key: bytes): + self.key = key + self.lock = threading.RLock() + + def verify(self, raw: Any) -> dict[str, Any]: + return verify_envelope(self.key, raw, expected_kind="assignment") + + def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]: + _payload, repo, branch, base = _binding(envelope) + destination = workspace_path(envelope) + state_path = _state_path(envelope) + with self.lock: + if (destination / ".git").exists(): + _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(state_path, 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else None + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + return {"workspace": str(destination), "baseline_sha": baseline} + if destination.exists() and any(destination.iterdir()): + raise ProtocolError("workspace is non-empty and unmanaged") + destination.parent.mkdir(parents=True, exist_ok=True) + broker = _broker_repo(repo) + try: + _run( + "clone", "--single-branch", "--branch", branch, "--no-tags", + broker, str(destination), timeout=900, + ) + except RuntimeError as error: + if destination.exists() and any(destination.iterdir()): + raise ProtocolError( + "failed branch checkout left unmanaged workspace state" + ) from error + if destination.exists(): + destination.rmdir() + _run( + "clone", "--single-branch", "--branch", base, "--no-tags", + broker, str(destination), timeout=900, + ) + _run("checkout", "-b", branch, cwd=destination) + _run( + "remote", "set-url", "origin", + f"https://scm.bstein.dev/atlas/{repo}.git", cwd=destination, + ) + _run("remote", "add", "hermes-broker", broker, cwd=destination) + _run("config", "user.name", "Hermes Execution Worker", cwd=destination) + _run("config", "user.email", "hermes@bstein.dev", cwd=destination) + baseline = _workspace_identity(destination, repo, branch) + atomic_json( + state_path, + {"baseline_sha": baseline, "repo": repo, "branch": branch}, + ) + return {"workspace": str(destination), "baseline_sha": baseline} + + @staticmethod + def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str: + query = urllib.parse.urlencode( + {"state": "open", "head": f"atlas:{branch}", "limit": 10} + ) + existing = json.loads( + scm_broker_client.read(f"/api/v1/repos/atlas/{repo}/pulls?{query}") + ) + if isinstance(existing, list) and existing: + return str(existing[0].get("html_url") or "") + created = json.loads( + scm_broker_client.create_draft( + repo, + base=base, + head=branch, + head_sha=head, + title=title, + body=body, + ) + ) + return str(created.get("html_url") or "") + + def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]: + """Enforce clean/committed state, broker push, and reviewed draft creation.""" + _payload, repo, branch, base = _binding(envelope) + destination = workspace_path(envelope) + title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip() + body = str(request.get("body") or "Automated Hermes draft.") + if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024: + raise ProtocolError("pull-request metadata exceeds the safe limit") + with self.lock: + head = _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(_state_path(envelope), 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else "" + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + status = _run( + "status", "--porcelain=v1", "--untracked-files=all", cwd=destination + ) + if status: + raise ProtocolError("workspace has uncommitted or untracked files") + ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination)) + if ahead <= 0: + return {"workspace": str(destination), "branch": branch, "pull_request": ""} + _run( + "push", "hermes-broker", f"HEAD:refs/heads/{branch}", + cwd=destination, timeout=900, + ) + pull = self._draft(repo, branch, base, head, title, body) + return {"workspace": str(destination), "branch": branch, "pull_request": pull} diff --git a/services/hermes/scripts/execution_pool_server.py b/services/hermes/scripts/execution_pool_server.py new file mode 100644 index 00000000..b2a01713 --- /dev/null +++ b/services/hermes/scripts/execution_pool_server.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Versioned HTTP and maintenance loop for the execution-pool coordinator.""" + +from __future__ import annotations + +import argparse +import os +import sqlite3 +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +from execution_pool_protocol import ( + MAX_WIRE_BYTES, + PROTOCOL_VERSION, + BoundedHTTPServer, + PoolStore, + ProtocolError, + canonical_json, + parse_wire, + read_key, +) + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +STATE_ROOT = DATA_ROOT / "execution-pool" +KEY_PATH = Path( + os.environ.get( + "HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key" + ) +) +PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007")) +RETENTION_SECONDS = int( + os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600") +) + + +def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + server_version = f"hermes-execution-pool/{PROTOCOL_VERSION}" + + def _reply(self, status: int, value: dict[str, Any]) -> None: + body = canonical_json(value) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + if self.path != "/ready": + self._reply(404, {"error": "not found"}) + return + try: + coordinator.store.available_ordinals() + self._reply(200, {"ready": True, "version": PROTOCOL_VERSION}) + except (OSError, sqlite3.Error): + self._reply(503, {"ready": False, "version": PROTOCOL_VERSION}) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > MAX_WIRE_BYTES: + raise ProtocolError("invalid content length") + envelope = parse_wire(self.rfile.read(length)) + routes = { + "/v1/poll": coordinator.poll, + "/v1/heartbeat": coordinator.heartbeat, + "/v1/result": coordinator.result, + } + if self.path not in routes: + self._reply(404, {"error": "not found"}) + return + self._reply(200, routes[self.path](envelope)) + except ProtocolError as error: + self._reply(409, {"error": str(error)}) + except Exception as error: + print( + f"pool request failed: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + self._reply(503, {"error": "coordinator unavailable"}) + + def log_message(self, _format: str, *_arguments: Any) -> None: + return + + return Handler + + +def run(coordinator_type: type[Any]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + store = PoolStore(STATE_ROOT / "assignments.db") + coordinator = coordinator_type(read_key(KEY_PATH), store) + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + ): + operation() + if args.once: + return 0 + server = BoundedHTTPServer( + ("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8 + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + while True: + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + lambda: store.garbage_collect(RETENTION_SECONDS), + ): + try: + operation() + except Exception as error: + print( + f"pool maintenance deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + time.sleep(5) diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py new file mode 100644 index 00000000..c98dfd56 --- /dev/null +++ b/services/hermes/scripts/execution_pool_worker.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import stat +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import cli_lane_runner +from execution_pool_protocol import ( + ProtocolError, + atomic_json, + canonical_json, +) + + +ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +CLIENT = os.environ.get( + "HERMES_EXECUTION_CLIENT_URL", + f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009", +).rstrip("/") +NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128] +RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600")) + + +def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]: + request = urllib.request.Request( + url, data=canonical_json(value), method="POST", + headers={"Content-Type": "application/json", "Cache-Control": "no-store"}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + body = response.read(64 * 1024 + 1) + if len(body) > 64 * 1024: + raise ProtocolError("server response exceeds the wire limit") + document = json.loads(body) + if not isinstance(document, dict): + raise ProtocolError("server response is not an object") + return document + + +def _client(operation: str, **values: Any) -> dict[str, Any]: + response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values}) + if response.get("error"): + raise ProtocolError(str(response["error"])) + return response + + +def _poll() -> dict[str, Any] | None: + assignment = _client("poll").get("assignment") + if assignment is None: + return None + if ( + not isinstance(assignment, dict) + or int(assignment["worker_ordinal"]) != ORDINAL + or int(assignment.get("protocol_version", 0)) != 2 + ): + raise ProtocolError("local boundary returned a foreign assignment") + return assignment + + +def _binding(assignment: dict[str, Any]) -> dict[str, Any]: + return {name: assignment[name] for name in ( + "board", "task_id", "run_id", "worker_ordinal", "attempt" + )} + + +def _state_path(assignment: dict[str, Any]) -> Path: + parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id")) + if any(not RUN_PART.fullmatch(part) for part in parts): + raise ProtocolError("assignment state path is invalid") + configured_root = ROOT / "session-state" + if configured_root.is_symlink(): + raise ProtocolError("assignment state root must not be a symlink") + root = configured_root.resolve() + path = root.joinpath(*parts).with_suffix(".json") + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent): + raise ProtocolError("assignment state path contains a symlink") + path.resolve(strict=False).relative_to(root) + return path + + +def _bind_provider_sessions(assignment: dict[str, Any]) -> None: + """Attach provider session directories to this exact durable task/run.""" + parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id")) + if any(not RUN_PART.fullmatch(part) for part in parts): + raise ProtocolError("provider session binding is invalid") + configured_root = ROOT / "provider-state" + if configured_root.is_symlink() or not configured_root.is_dir(): + raise ProtocolError("durable provider state root is unavailable") + provider_root = configured_root.resolve() + run_root = provider_root + for part in parts: + run_root /= part + if run_root.is_symlink(): + raise ProtocolError("provider session path contains a symlink") + run_root.mkdir(mode=0o700, exist_ok=True) + run_root.resolve().relative_to(provider_root) + home = run_root / "home" + if home.is_symlink(): + raise ProtocolError("provider HOME must not be a durable symlink") + home.mkdir(mode=0o700, parents=True, exist_ok=True) + runtime_home = cli_lane_runner.DATA_ROOT / "home" + if runtime_home.is_symlink(): + runtime_home.unlink() + elif runtime_home.exists(): + raise ProtocolError("provider HOME is not a task-bound symlink") + runtime_home.symlink_to(home) + (home / ".claude").mkdir(mode=0o700, exist_ok=True) + settings = home / ".claude/settings.json" + if not settings.exists(): + atomic_json(settings, {}, 0o600) + bindings = { + Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "sessions": + run_root / "codex/sessions", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "projects": + run_root / "claude/projects", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "session-env": + run_root / "claude/session-env", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "todos": + run_root / "claude/todos", + } + for runtime, durable in bindings.items(): + current = run_root + for part in durable.relative_to(run_root).parts: + current /= part + if current.is_symlink(): + raise ProtocolError("provider session path contains a symlink") + current.mkdir(mode=0o700, exist_ok=True) + current.resolve().relative_to(provider_root) + if runtime.is_symlink(): + runtime.unlink() + elif runtime.exists(): + raise ProtocolError(f"provider session path is not a symlink: {runtime.name}") + runtime.symlink_to(durable) + + +def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str: + return f"""You are a durable coding worker managed by Hermes Kanban. + +Work only on this objective and its acceptance criteria: + +{context} + +Workspace: {workspace} +Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL} + +Operate autonomously only inside this private assigned checkout. Inspect before editing, +preserve unrelated and untracked files, and fail closed rather than overwrite state you do +not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to +read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots, or bypass +the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch; +the worker boundary handles the bounded push and draft pull request after validation. +Switchyard owns provider/model/effort selection and cross-provider fallback. + +Return a final JSON object matching the supplied schema. Use status=incomplete when work, +tests, commits, or verification remain. Use blocked only for a concrete task obstacle. +Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers. +""" + + +def _read_activity(log_path: Path, offset: int) -> tuple[str, int]: + flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(log_path, flags) + except FileNotFoundError: + return "", offset + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise ProtocolError("worker log must be a regular file") + if info.st_size < offset: + offset = 0 + os.lseek(descriptor, offset, os.SEEK_SET) + value = os.read(descriptor, 12 * 1024) + next_offset = os.lseek(descriptor, 0, os.SEEK_CUR) + finally: + os.close(descriptor) + return value.decode("utf-8", "replace"), next_offset + + +def _git(workspace: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL, + text=True, capture_output=True, timeout=30, check=False, + env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "/bin/false"}, + ) + if completed.returncode: + raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:]) + return completed.stdout.strip() + + +def _bounded_result(value: dict[str, Any]) -> dict[str, Any]: + """Keep terminal evidence useful while fitting the authenticated wire cap.""" + bounded: dict[str, Any] = {} + for name in ("status", "summary"): + bounded[name] = str(value.get(name) or "")[:8000] + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + items = value.get(name) + bounded[name] = [str(item)[:2000] for item in items[:100]] if isinstance(items, list) else [] + while len(canonical_json(bounded)) > 32 * 1024: + longest = max( + (name for name in bounded if isinstance(bounded[name], list) and bounded[name]), + key=lambda name: len(canonical_json(bounded[name])), + default="", + ) + if longest: + bounded[longest].pop() + else: + bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2] + return bounded + + +def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]: + fresh = _poll() + if fresh is None or _binding(fresh) != binding: + raise ProtocolError("assignment changed before SCM submission") + return fresh + + +def garbage_collect(now: float | None = None) -> int: + """Remove only clean, terminal, assignment-derived workspaces after retention.""" + current = time.time() if now is None else now + state_root = (ROOT / "session-state").resolve() + run_root = (ROOT / "runs").resolve() + removed = 0 + for state_file in state_root.glob("*/*/*.json") if state_root.is_dir() else (): + state = cli_lane_runner.load_json(state_file) + terminal_at = float(state.get("terminal_at") or 0) + if terminal_at <= 0 or current - terminal_at < max(3600, RETENTION_SECONDS): + continue + workspace = Path(str(state.get("workspace") or "")) + try: + workspace.resolve(strict=True).relative_to(run_root) + except (OSError, RuntimeError, ValueError): + continue + if workspace.is_symlink() or _git( + workspace, "status", "--porcelain=v1", "--untracked-files=all" + ): + continue + shutil.rmtree(workspace) + state_file.unlink() + removed += 1 + return removed + + +def execute(assignment: dict[str, Any]) -> None: + binding = _binding(assignment) + payload = assignment["payload"] + if not isinstance(payload, dict): + raise ProtocolError("assignment payload is invalid") + _bind_provider_sessions(assignment) + workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True) + workspace.relative_to((ROOT / "runs").resolve()) + state_file = _state_path(assignment) + state = cli_lane_runner.load_json(state_file) + state.update({**binding, "node": NODE, "workspace": str(workspace)}) + state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or "")) + atomic_json(state_file, state) + log_path = state_file.with_suffix(".log") + offset = 0 + latest_route: dict[str, Any] = {} + + def heartbeat(note: str) -> bool: + nonlocal offset + try: + activity, offset = _read_activity(log_path, offset) + response = _client( + "heartbeat", binding=binding, + payload={ + "note": f"worker={ORDINAL} node={NODE} {note}"[:1000], + "activity": activity, "route": latest_route, + }, + ) + return bool(response.get("ack", {}).get("accepted")) + except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): + return False + + context = str(payload.get("context") or "") + assignee = str(payload.get("assignee") or "cli-auto") + route = cli_lane_runner.select_route( + context, assignee, + exclude_provider=cli_lane_runner.fresh_unavailable_provider() + if assignee == "cli-auto" else None, + ) + latest_route = asdict(route) + if not heartbeat( + f"route={route.provider}/{route.model}/{route.effort}; assignment accepted" + ): + raise ProtocolError("assignment lease was lost before provider startup") + deadline = min( + int(payload.get("deadline_unix") or int(time.time()) + 60), + int(time.time()) + int(payload.get("max_runtime_seconds") or 60), + ) + result = cli_lane_runner.run_provider( + route, _prompt(context, workspace, binding), workspace, state, state_file, + log_path, heartbeat, max(1, deadline - int(time.time())), + ) + if result.capacity_failure and deadline - time.time() > 30: + alternate = "claude" if route.provider == "codex" else "codex" + fallback = cli_lane_runner.select_route( + context + "\nThe first provider failed from capacity or authentication.", + f"cli-{alternate}-{route.effort}", + ) + latest_route = asdict(fallback) + heartbeat(f"fallback={route.provider}->{fallback.provider}") + result = cli_lane_runner.run_provider( + fallback, + _prompt(context, workspace, binding) + + cli_lane_runner.git_handoff(workspace, result.output), + workspace, state, state_file, log_path, heartbeat, + max(1, deadline - int(time.time())), + ) + route = fallback + structured = dict(result.structured or {}) + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + if not isinstance(structured.get(name), list): + structured[name] = [] + if structured.get("status") == "completed" and result.returncode == 0: + status = _git(workspace, "status", "--porcelain=v1", "--untracked-files=all") + if status: + structured["status"] = "incomplete" + structured["blockers"].append( + "Worker left uncommitted or untracked files; SCM submission failed closed." + ) + else: + baseline = str(state.get("baseline_sha") or "") + if not re.fullmatch(r"[0-9a-f]{40,64}", baseline): + raise RuntimeError("durable SCM baseline is missing or invalid") + _refresh_assignment(binding) + structured = _bounded_result(structured) + activity, _ = _read_activity(log_path, offset) + result_payload = { + "structured": structured, + "returncode": result.returncode, + "capacity_failure": result.capacity_failure, + "node": NODE, + "route": asdict(route), + "provider_sessions": { + "codex_thread_id": state.get("codex_thread_id"), + "claude_session_id": state.get("claude_session_id"), + }, + "final_activity": activity, + } + response = _client( + "finish", + binding=binding, + payload=result_payload, + title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240], + body=json.dumps(structured, indent=2, sort_keys=True)[:12000], + ) + if not response.get("ack", {}).get("accepted"): + raise ProtocolError("coordinator did not accept the terminal result") + state["terminal_at"] = time.time() + atomic_json(state_file, state) + + +def report_exception(assignment: dict[str, Any], error: Exception) -> bool: + """Surface one worker exception as an exact-run transient result.""" + binding = _binding(assignment) + detail = f"{type(error).__name__}: {error}"[:1500] + structured = { + "status": "blocked", + "summary": "Distributed worker failed before a safe handoff.", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [detail], + } + payload = { + "structured": structured, + "returncode": 1, + # This is an execution-infrastructure failure, not an objective capability + # verdict. The exact-run result releases the ordinal for another task. + "capacity_failure": True, + "node": NODE, + "route": {}, + "provider_sessions": {}, + "final_activity": detail, + } + try: + return bool( + _client("finish", binding=binding, payload=payload).get("ack", {}).get( + "accepted" + ) + ) + except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): + return False + + +def readiness() -> None: + if ORDINAL not in range(3): + raise ProtocolError("worker ordinal must be 0, 1, or 2") + for path in (ROOT, ROOT / "provider-state", cli_lane_runner.DATA_ROOT): + if not path.is_dir() or not os.access(path, os.W_OK): + raise ProtocolError(f"durable worker path is not writable: {path}") + for credential in ( + Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "auth.json", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / ".credentials.json", + ): + if not credential.is_file() or not os.access(credential, os.W_OK): + raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}") + cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + atomic_json( + cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644 + ) + _poll() + + +def main() -> int: + readiness() + while True: + assignment = None + try: + garbage_collect() + assignment = _poll() + if assignment is None: + time.sleep(5) + continue + execute(assignment) + except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error: + surfaced = assignment is not None and report_exception(assignment, error) + print( + f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: " + f"{type(error).__name__}: {error}", + flush=True, + ) + time.sleep(10) + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index 07423d70..2d998dd8 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -4,53 +4,130 @@ from __future__ import annotations import argparse +import hashlib +import hmac import json import os +import stat +import uuid from pathlib import Path VAULT_ROOT = Path("/vault/secrets") RUNTIME_ROOT = Path("/runtime-access") PERSISTENT_HOME = Path("/opt/data/home") +WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +PROVIDER_ACCESS_ROOT = Path( + os.environ.get("HERMES_PROVIDER_ACCESS_ROOT", "/provider-access") +) +POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access")) OWNER_UID = 10000 OWNER_GID = 10000 def _owned_directory(path: Path) -> None: """Create one private directory owned by the unprivileged Hermes user.""" + if path.is_symlink(): + raise RuntimeError(f"private directory {path.name} must not be a symlink") path.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.is_symlink() or not path.is_dir(): + raise RuntimeError(f"private directory {path.name} is invalid") path.chmod(0o700) os.chown(path, OWNER_UID, OWNER_GID) -def _copy_secret(source_name: str, destination: Path) -> str: - """Copy a non-empty Vault projection without logging its value.""" - value = (VAULT_ROOT / source_name).read_text(encoding="utf-8").strip() +def _read_secret(source_name: str) -> str: + path = VAULT_ROOT / source_name + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= 1024 * 1024: + raise RuntimeError(f"Vault projection {source_name} is invalid") + raw = os.read(descriptor, info.st_size + 1) + finally: + os.close(descriptor) + value = raw.decode("utf-8").strip() if not value: raise RuntimeError(f"Vault projection {source_name} is empty") - destination.write_text(value + "\n", encoding="utf-8") - destination.chmod(0o600) - os.chown(destination, OWNER_UID, OWNER_GID) return value +def _write_secret(destination: Path, value: str) -> None: + temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + remaining = memoryview((value + "\n").encode()) + while remaining: + remaining = remaining[os.write(descriptor, remaining) :] + os.fsync(descriptor) + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, OWNER_UID, OWNER_GID) + finally: + os.close(descriptor) + try: + os.replace(temporary, destination) + finally: + temporary.unlink(missing_ok=True) + + +def _copy_secret(source_name: str, destination: Path) -> str: + """Copy a non-empty Vault projection without logging its value.""" + value = _read_secret(source_name) + _write_secret(destination, value) + return value + + +def _validate_json_value(value: str, source_name: str, required: tuple[str, ...]) -> None: + """Validate a provider credential shape without exposing its value.""" + try: + document = json.loads(value) + except json.JSONDecodeError as error: + raise RuntimeError(f"credential {source_name} is not valid JSON") from error + current = document + for key in required: + if not isinstance(current, dict) or key not in current: + raise RuntimeError(f"credential {source_name} has an invalid shape") + current = current[key] + if not isinstance(current, str) or not current: + raise RuntimeError(f"credential {source_name} has an empty credential") + + def _validated_json(source_name: str, destination: Path, required: tuple[str, ...]) -> None: """Stage one credential document after checking its expected shape.""" value = _copy_secret(source_name, destination) try: - document = json.loads(value) - except json.JSONDecodeError: + _validate_json_value(value, source_name, required) + except RuntimeError: destination.unlink(missing_ok=True) raise - current = document - for key in required: - if not isinstance(current, dict) or key not in current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an invalid shape") - current = current[key] - if not isinstance(current, str) or not current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an empty credential") + + +def _bootstrap_json( + source_name: str, destination: Path, required: tuple[str, ...] +) -> None: + """Initialize one ordinal credential once, then preserve provider refreshes.""" + if destination.is_symlink(): + raise RuntimeError(f"durable credential {destination.name} must not be a symlink") + try: + descriptor = os.open( + destination, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + ) + except FileNotFoundError: + _validated_json(source_name, destination, required) + return + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_mode & 0o077 + or not 0 < info.st_size <= 1024 * 1024 + ): + raise RuntimeError(f"durable credential {destination.name} is unsafe") + value = os.read(descriptor, info.st_size + 1).decode("utf-8").strip() + finally: + os.close(descriptor) + _validate_json_value(value, destination.name, required) def _link_noncredential_state( @@ -107,6 +184,7 @@ def stage_agent() -> None: _owned_directory(path) for name in ( "agent-api-key", + "execution-pool-key", "chat-relay-key", "node-ssh-private-key", "node-ssh-known-hosts", @@ -157,16 +235,77 @@ def stage_triage() -> None: _write_empty_auth_store() +def _durable_link(runtime: Path, durable: Path, name: str) -> None: + """Link one provider state path only after rejecting replacement symlinks.""" + durable.mkdir(mode=0o700, parents=True, exist_ok=True) + durable.chmod(0o700) + os.chown(durable, OWNER_UID, OWNER_GID) + target = runtime / name + if target.is_symlink() and target.resolve(strict=False) != durable.resolve(): + target.unlink() + elif target.exists() and not target.is_symlink(): + raise RuntimeError(f"refusing to replace provider state path: {target}") + if not target.exists(): + target.symlink_to(durable) + + +def stage_execution_worker() -> None: + """Bootstrap and preserve ordinal-private subscription refresh ownership.""" + provider_state = WORKER_ROOT / "provider-state" + for path in ( + provider_state, + PROVIDER_ACCESS_ROOT, + PROVIDER_ACCESS_ROOT / "claude", + PROVIDER_ACCESS_ROOT / "codex", + ): + _owned_directory(path) + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution worker ordinal is invalid") + _bootstrap_json( + f"claude-credentials-{ordinal}", + PROVIDER_ACCESS_ROOT / "claude" / ".credentials.json", + ("claudeAiOauth", "refreshToken"), + ) + _bootstrap_json( + f"codex-auth-{ordinal}", + PROVIDER_ACCESS_ROOT / "codex" / "auth.json", + ("tokens", "refresh_token"), + ) + + +def stage_execution_mediator() -> None: + """Derive one ordinal HMAC authority in the isolated mediator Pod only.""" + _owned_directory(POOL_ACCESS_ROOT) + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution mediator ordinal is invalid") + master = _read_secret("execution-pool-key").encode() + derived = hmac.new( + master, + f"hermes-execution-pool-v2:worker:{ordinal}".encode(), + hashlib.sha256, + ).hexdigest() + _write_secret(POOL_ACCESS_ROOT / "execution-pool-key", derived) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("agent", "chat", "triage")) + parser.add_argument( + "mode", + choices=("agent", "chat", "triage", "execution-worker", "execution-mediator"), + ) args = parser.parse_args() if args.mode == "agent": stage_agent() elif args.mode == "chat": stage_chat() - else: + elif args.mode == "triage": stage_triage() + elif args.mode == "execution-worker": + stage_execution_worker() + else: + stage_execution_mediator() print("Runtime access staged from Vault.", flush=True) return 0 diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index 0fc8d939..3903b5f9 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -117,6 +117,37 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app: hermes-execution-worker +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: hermes-execution-worker +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-execution-pool + namespace: hermes + labels: + app: hermes-agent +spec: + type: ClusterIP + publishNotReadyAddresses: true + selector: + app: hermes-agent + ports: + - name: http + port: 9007 + targetPort: execution-pool + protocol: TCP +--- +apiVersion: v1 +kind: Service metadata: name: hermes-local-image namespace: hermes diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 612391c9..d6baabc5 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -223,6 +223,8 @@ write_policy_and_role "outline" "outline" "outline-vault" \ "outline/* shared/postmark-relay" "" write_policy_and_role "planka" "planka" "planka-vault" \ "planka/* shared/postmark-relay" "" +write_policy_and_role "hermes-execution-worker" "hermes" "hermes-execution-worker" \ + "hermes/agent-tokens" "" write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home,bstein-dev-home-vault-sync" \ "portal/* shared/chat-ai-keys-runtime shared/portal-e2e-client shared/postmark-relay mailu/mailu-initial-account-secret shared/harbor-pull" "" write_policy_and_role "gitea" "gitea" "gitea-vault" \ diff --git a/testing/tests/test_hermes_agent_access.py b/testing/tests/test_hermes_agent_access.py index d4138256..a932c56c 100644 --- a/testing/tests/test_hermes_agent_access.py +++ b/testing/tests/test_hermes_agent_access.py @@ -78,15 +78,16 @@ def test_owner_agent_has_pinned_dedicated_node_ssh_access(): "secretProviderClass": "hermes-node-ssh-access" } reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "grep -qxF" not in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = next(item for item in pod["volumes"] if item["name"] == "coordinator") + assert hardener["configMap"] == { + "name": "hermes-node-account-hardener", + "defaultMode": 0o555, + } def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): @@ -155,13 +156,16 @@ def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): rbac = [ item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) + for item in yaml.safe_load_all((HERMES / "rbac.yaml").read_text()) if item ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] + bindings = [item for item in rbac if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all( + subject["name"] not in {"hermes-agent", "hermes-switchyard"} + for binding in bindings + for subject in binding["subjects"] + ) def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py index cf23a2d0..f3410035 100644 --- a/testing/tests/test_hermes_agent_security.py +++ b/testing/tests/test_hermes_agent_security.py @@ -182,21 +182,54 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces() "ports": [{"protocol": "TCP", "port": 9010}], }, ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + egress = isolation["spec"]["egress"] + assert {} not in egress + broker = next( + rule + for rule in egress + if rule.get("ports") == [{"protocol": "TCP", "port": 9081}] + ) + assert broker["to"] == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "hermes-scm"} + }, + "podSelector": {"matchLabels": {"app": "hermes-scm-broker"}}, + } ] + namespace_rule = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("namespaceSelector", {}).get( + "matchExpressions" + ) + ) + assert namespace_rule["to"][0]["namespaceSelector"]["matchExpressions"] == [ + { + "key": "kubernetes.io/metadata.name", + "operator": "NotIn", + "values": ["gitea", "hermes-scm"], + } + ] + public = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("ipBlock", {}).get("cidr") == "0.0.0.0/0" + ) + assert "192.168.0.0/16" in public["to"][0]["ipBlock"]["except"] + + +def test_owner_agent_has_observer_kubernetes_context_without_cluster_admin(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = HERMES / "rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + bindings = [item for item in documents if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all(binding["roleRef"]["name"] != "cluster-admin" for binding in bindings) + assert all( + subject["name"] != "hermes-agent" + for binding in bindings + for subject in binding["subjects"] + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..05af3b01 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,6 +255,7 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) + monkeypatch.setattr(module, "_fresh_health", lambda _path: {}) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_chat_config.py b/testing/tests/test_hermes_chat_config.py index 7dd0e21c..c964e3e1 100644 --- a/testing/tests/test_hermes_chat_config.py +++ b/testing/tests/test_hermes_chat_config.py @@ -56,16 +56,17 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "Atlas Git access is supplied by the isolated SCM\nbroker" in soul assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "no repository token is present in this pod" in soul + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "Never load or follow a\nGitHub/`gh`" in instructions + assert "use `/opt/scm/gitea_api.py`" in instructions + assert "never bypass the client with raw HTTP" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_chat_provider_runtime.py b/testing/tests/test_hermes_chat_provider_runtime.py new file mode 100644 index 00000000..ebafe2ad --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_runtime.py @@ -0,0 +1,196 @@ +"""Native provider health, API lineage, and runtime-image contracts.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, + _load_broker_module, +) + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert 'detail = tool_name if event_type in {' in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert 'heartbeats.get(session_id, 0.0)' in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" + assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) diff --git a/testing/tests/test_hermes_chat_voice.py b/testing/tests/test_hermes_chat_voice.py index 180a17e4..e04f3908 100644 --- a/testing/tests/test_hermes_chat_voice.py +++ b/testing/tests/test_hermes_chat_voice.py @@ -254,11 +254,14 @@ def test_chat_image_generation_uses_private_owner_broker(): vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() assert ( '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' + 'hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins ' 'hermes/developer-ssh"' in vault_policy ) + assert ( + 'write_policy_and_role "hermes-scm-broker" "hermes-scm" ' + '"hermes-scm-broker"' in vault_policy + ) assert ( 'write_policy_and_role "hermes-node-ssh" "hermes" ' '"hermes-node-ssh-access"' in vault_policy diff --git a/testing/tests/test_hermes_cli_capabilities.py b/testing/tests/test_hermes_cli_capabilities.py index adc6e3ca..ddb67492 100644 --- a/testing/tests/test_hermes_cli_capabilities.py +++ b/testing/tests/test_hermes_cli_capabilities.py @@ -93,6 +93,9 @@ def test_capability_detection_requires_explicit_safety_keywords(): assert lanes.detect_kanban_capabilities(variadic).ready is False assert lanes._explicit_keyword(object(), "expected_run_id") is False + missing_completion = lanes.KanbanCapabilities(False, True, True) + assert missing_completion.deferred_features == ("exact-run-completion",) + def test_startup_health_moves_from_bounded_deferred_to_ready( tmp_path: Path, @@ -117,9 +120,10 @@ def test_startup_health_moves_from_bounded_deferred_to_ready( assert "compatibility deferred" in capsys.readouterr().err new_db = _db(patched=True) - new = lanes.kanban_capabilities(new_db) - lanes.initialize_kanban_capabilities(new_db, health_path=health) + lanes.kanban_capabilities(new_db) + new = lanes.initialize_kanban_capabilities(new_db, health_path=health) assert new.ready is True + assert lanes.kanban_capabilities(new_db) is new assert lanes.runtime_health(health)["state"] == "ready" diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py index 74684105..8355b8c0 100644 --- a/testing/tests/test_hermes_cli_dispatch_runtime.py +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -13,6 +13,8 @@ from testing.tests.test_hermes_cli_support import lanes def test_board_slug_and_connection_failure_paths(monkeypatch): + assert lanes._owns_local_workspace(SimpleNamespace(workspace_path=" /workspace ")) + assert not lanes._owns_local_workspace(SimpleNamespace(workspace_path="")) assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra" assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback" failures = [] @@ -100,6 +102,41 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc assert lanes.claim_ready(set(), 1) == [] +def test_dispatch_skips_unhealthy_ineligible_and_unclaimed_entries(monkeypatch): + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + tasks = [ + SimpleNamespace(id="t_ineligible", status="ready", assignee="cli-auto"), + SimpleNamespace(id="t_unclaimed", status="ready", assignee="cli-auto"), + ] + db = SimpleNamespace( + list_boards=lambda **_kwargs: [ + {"slug": "offline"}, + {"slug": "cassandra"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (_ for _ in ()).throw(OSError("offline")) + if board == "offline" + else Connection(board), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: tasks, + claim_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + assert lanes.claim_ready( + set(), + 2, + eligible=lambda _board, task: task.id == "t_unclaimed", + ) == [] + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + lanes.recover_orphans() + + class _Future: def __init__(self): self.polls = 0 diff --git a/testing/tests/test_hermes_cli_evidence_edges.py b/testing/tests/test_hermes_cli_evidence_edges.py index c28ff43f..6a954864 100644 --- a/testing/tests/test_hermes_cli_evidence_edges.py +++ b/testing/tests/test_hermes_cli_evidence_edges.py @@ -44,6 +44,7 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( path = tmp_path / "pending" path.write_text("old", encoding="utf-8") source = path.stat() + source_descriptor = os.open(path, os.O_RDONLY) directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: path.unlink() @@ -52,6 +53,7 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "missing" path.write_text("new", encoding="utf-8") @@ -60,8 +62,11 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "replacement" + os.close(source_descriptor) + source_descriptor = os.open(path, os.O_RDONLY) replacement = path.stat() monkeypatch.setattr( lanes, @@ -73,8 +78,10 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( replacement, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "collision" finally: + os.close(source_descriptor) os.close(directory) @@ -82,6 +89,7 @@ def test_retirement_preserves_a_postcheck_replacement(tmp_path: Path, monkeypatc path = tmp_path / "pending" path.write_text("old", encoding="utf-8") source = path.stat() + source_descriptor = os.open(path, os.O_RDONLY) directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) calls = [] @@ -102,8 +110,10 @@ def test_retirement_preserves_a_postcheck_replacement(tmp_path: Path, monkeypatc source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "replacement" finally: + os.close(source_descriptor) os.close(directory) assert path.read_text(encoding="utf-8") == "replacement" @@ -133,6 +143,31 @@ def test_evidence_writer_bounds_payload_and_cleans_failed_temporary( monkeypatch.setattr(lanes, "_retire_terminal_entry", real_retire) +def test_evidence_writer_closes_unwrapped_and_rejects_unsafe_temporary_files( + tmp_path: Path, + monkeypatch, +): + destination = tmp_path / "evidence.json" + real_fdopen = lanes.os.fdopen + opened = [] + + def fail_fdopen(descriptor, *_args, **_kwargs): + opened.append(descriptor) + raise OSError("fdopen failed") + + monkeypatch.setattr(lanes.os, "fdopen", fail_fdopen) + with pytest.raises(OSError, match="fdopen failed"): + lanes._write_json_noreplace(destination, {"value": "bounded"}) + with pytest.raises(OSError): + os.fstat(opened[0]) + + monkeypatch.setattr(lanes.os, "fdopen", real_fdopen) + with monkeypatch.context() as unsafe: + unsafe.setattr(lanes.stat, "S_IMODE", lambda _mode: 0o644) + with pytest.raises(OSError, match="not private"): + lanes._write_json_noreplace(destination, {"value": "bounded"}) + + def test_evidence_paths_and_collision_validation_fail_closed( tmp_path: Path, monkeypatch, diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py index 07b9035d..889218db 100644 --- a/testing/tests/test_hermes_cli_execution_edges.py +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -16,6 +16,13 @@ class _Connection: return None +def test_goal_rejection_history_is_list_bounded(): + execution = __import__("cli_lane_execution") + history = ["first"] + assert execution._goal_rejections({"goal_rejections": history}) is history + assert execution._goal_rejections({"goal_rejections": "invalid"}) == [] + + def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch): db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), @@ -64,12 +71,18 @@ def test_transient_callback_storage_errors_do_not_kill_provider( def provider(*args, **_kwargs): assert args[6]("still working") is True - return lanes.ProcessResult(1, "failed", None, False) + return lanes.ProcessResult( + 1, + "failed", + {"status": "incomplete", "blockers": ["synthetic provider failure"]}, + False, + ) monkeypatch.setattr(lanes, "run_provider", provider) lanes.execute_claim("cassandra", "t_callbacks") assert blocks and blocks[0]["kind"] == "capability" + assert blocks[0]["reason"] == "synthetic provider failure" def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch): diff --git a/testing/tests/test_hermes_cli_fallback.py b/testing/tests/test_hermes_cli_fallback.py index e2a8141e..728c8fe7 100644 --- a/testing/tests/test_hermes_cli_fallback.py +++ b/testing/tests/test_hermes_cli_fallback.py @@ -53,6 +53,10 @@ def test_goal_card_continues_after_local_judge_rejects_progress( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + lanes.atomic_json( + lanes.state_path("cassandra", "t_goal"), + {"goal_rejections": ["prior incomplete report"]}, + ) claude_low = lanes.Route( "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () ) diff --git a/testing/tests/test_hermes_cli_finalization_edges.py b/testing/tests/test_hermes_cli_finalization_edges.py index 0f2c2a2c..f8210243 100644 --- a/testing/tests/test_hermes_cli_finalization_edges.py +++ b/testing/tests/test_hermes_cli_finalization_edges.py @@ -159,6 +159,14 @@ def test_terminal_finalizer_closes_supplied_invalid_snapshots( monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") path = lanes._terminal_path(lanes.state_path("board", "task"), 8) + lanes.atomic_json(path, {}) empty = _Snapshot(None) assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid" assert empty.closed == 1 + + finalization = __import__("cli_lane_finalization") + direct = _Snapshot(None) + assert finalization._finalize_terminal_record( + _db(None), path, snapshot=direct + ) == "invalid" + assert direct.closed == 1 diff --git a/testing/tests/test_hermes_cli_foundation_coverage.py b/testing/tests/test_hermes_cli_foundation_coverage.py index 71f0c44a..83fda691 100644 --- a/testing/tests/test_hermes_cli_foundation_coverage.py +++ b/testing/tests/test_hermes_cli_foundation_coverage.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import runpy import sqlite3 from contextlib import nullcontext from pathlib import Path @@ -11,7 +13,7 @@ from urllib.error import URLError import pytest -from testing.tests.test_hermes_cli_support import _completed_result, lanes +from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes def test_board_context_serializes_objects_and_storage_failure_is_bounded( @@ -93,6 +95,26 @@ def test_json_and_terminal_path_helpers_fail_closed(tmp_path: Path, monkeypatch) ) assert lanes._terminal_evidence_identity(missing_evidence) is None + with pytest.raises(ValueError, match="positive SQLite"): + lanes.TerminalIdentity("board", "task", 0, "pending") + + +def test_recovery_snapshot_close_is_idempotent_after_descriptors_are_released( + tmp_path: Path, +): + directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + snapshot = lanes.TerminalRecoverySnapshot( + None, + tmp_path.stat(), + None, + directory, + b"", + "synthetic", + ) + snapshot.close() + snapshot.close() + assert snapshot.directory_descriptor == -1 + def test_goal_helpers_cover_compaction_and_deterministic_failures(): compacted = lanes.cli_lane_goal._bounded("a" * 100, 40) @@ -122,6 +144,9 @@ def test_goal_helpers_cover_compaction_and_deterministic_failures(): ) assert accepted is False assert reason + assert lanes.cli_lane_goal.unfinished_result_reason( + {"status": "completed", "summary": "done", "tests_run": "not-a-list"} + ) is None def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs( @@ -147,6 +172,31 @@ def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs( assert lanes._extract_json('prefix {"status":"blocked"} suffix') == { "status": "blocked" } + assert lanes._extract_json('{"status":"unknown"}') is None + + state_file = workspace / "state.json" + state = {} + assert lanes._event_payload( + "codex", + json.dumps({"type": "thread.started", "result": {"status": "blocked"}}), + state, + state_file, + ) == {"status": "blocked"} + assert lanes._event_payload( + "claude", + json.dumps({"session_id": "session-2", "text": "not-json"}), + state, + state_file, + ) is None + assert state["claude_session_id"] == "session-2" + + prompt_module = __import__("cli_lane_prompt") + assert prompt_module.workspace_artifacts( + workspace, [str(directory), str(artifact)] + ) == [str(artifact.resolve())] + assert prompt_module._event_payload( + "other", '{"item": {}}', {}, state_file + ) is None def test_provider_event_parsing_persists_sessions_and_nested_results( @@ -205,3 +255,11 @@ def test_switchyard_network_failure_is_explicit(): URLError("offline") ), ) + + +def test_runner_main_guard_delegates_to_dispatch(monkeypatch): + dispatch = __import__("cli_lane_dispatch") + monkeypatch.setattr(dispatch, "main", lambda: 17) + with pytest.raises(SystemExit) as raised: + runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__") + assert raised.value.code == 17 diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py index bf1807f4..8489be89 100644 --- a/testing/tests/test_hermes_cli_lanes_configuration.py +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -20,6 +20,8 @@ from testing.tests.test_hermes_cli_lanes_support import ( policy, ) +import cli_lane_execution as lane_execution + def test_workspace_preparation_failure_durably_blocks_the_claim( tmp_path: Path, monkeypatch @@ -43,7 +45,7 @@ def test_workspace_preparation_failure_durably_blocks_the_claim( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + lane_execution, "state_path", lambda _board, _task_id: tmp_path / "state.json" ) lanes.execute_claim("cassandra", "t_bad_worktree") @@ -102,21 +104,21 @@ def test_restart_provider_change_includes_explicit_workspace_handoff( block_task=lambda *_args, **_kwargs: None, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr(lane_execution, "state_path", lambda _board, _task_id: state_file) monkeypatch.setattr( - lanes, + lane_execution, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () ), ) monkeypatch.setattr( - lanes, + lane_execution, "git_handoff", lambda _workspace, output: f"HANDOFF:{output}", ) monkeypatch.setattr( - lanes, + lane_execution, "run_provider", lambda _route, prompt, *_args, **_kwargs: ( prompts.append(prompt) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py deleted file mode 100644 index 279b74c5..00000000 --- a/testing/tests/test_hermes_cli_lanes_kanban.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" - -from __future__ import annotations - -import sys -from contextlib import nullcontext -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from testing.tests.test_hermes_cli_lanes_support import ( - lanes, -) - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( - monkeypatch, capsys -): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" - ) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any( - "Goal completion rejected; continuing turn 2/3" in item for item in comments - ) - assert any( - "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments - ) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] diff --git a/testing/tests/test_hermes_cli_provider_edges.py b/testing/tests/test_hermes_cli_provider_edges.py index 21e57453..6f6f88a3 100644 --- a/testing/tests/test_hermes_cli_provider_edges.py +++ b/testing/tests/test_hermes_cli_provider_edges.py @@ -44,6 +44,26 @@ def test_stream_process_enforces_runtime_and_lease_boundaries( assert lease_lost.returncode != 0 +def test_stream_process_bounds_retained_line_history(tmp_path: Path): + result = lanes.stream_process( + [ + sys.executable, + "-c", + "import sys; [sys.stdout.write(f'{n}\\n') for n in range(4010)]", + ], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "bounded.log", + state={}, + state_file=tmp_path / "state.json", + heartbeat=lambda _note: True, + max_runtime=60, + ) + assert result.returncode == 0 + assert "4009" in result.output + + def test_process_identity_and_signal_failures_are_bounded(monkeypatch): assert lanes._process_record(999999999) is None monkeypatch.setattr( @@ -63,9 +83,26 @@ def test_process_identity_and_signal_failures_are_bounded(monkeypatch): ) lanes._signal_worker_tree(10, {11: (12, 13)}, signal.SIGTERM) + signals = [] + monkeypatch.setattr(lanes, "_process_identity_matches", lambda *_args: False) + monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig))) + monkeypatch.setattr( + lanes.os, + "kill", + lambda *_args: (_ for _ in ()).throw(AssertionError("stale PID signaled")), + ) + lanes._signal_worker_tree(20, {21: (22, 23)}, signal.SIGKILL) + assert signals == [(20, signal.SIGKILL)] + def test_descendant_snapshot_follows_multiple_generations(monkeypatch): - entries = [Path("/proc/self"), Path("/proc/10"), Path("/proc/11"), Path("/proc/12")] + entries = [ + Path("/proc/self"), + Path("/proc/10"), + Path("/proc/11"), + Path("/proc/12"), + Path("/proc/13"), + ] monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) records = { 10: (1, 10, 100), @@ -76,6 +113,34 @@ def test_descendant_snapshot_follows_multiple_generations(monkeypatch): assert lanes._descendant_processes(10) == {11: (11, 101), 12: (12, 102)} +def test_claude_missing_unstarted_session_does_not_mark_started( + tmp_path: Path, + monkeypatch, +): + state = {"claude_session_id": "synthetic-session"} + state_file = tmp_path / "state.json" + monkeypatch.setattr( + lanes, + "stream_process", + lambda *_args, **_kwargs: lanes.ProcessResult( + 1, lanes.NO_CLAUDE_SESSION, None, False + ), + ) + route = lanes.Route("claude", "model", "high", "profile", "test", "test", 1, ()) + result = lanes.run_provider( + route, + "work", + tmp_path, + state, + state_file, + tmp_path / "log", + lambda _note: True, + 60, + ) + assert result.returncode == 1 + assert "claude_started" not in state + + def test_terminate_escalates_after_term_timeout(monkeypatch): class Process: pid = 321 diff --git a/testing/tests/test_hermes_cli_records_edges.py b/testing/tests/test_hermes_cli_records_edges.py index 3362466a..fed9e872 100644 --- a/testing/tests/test_hermes_cli_records_edges.py +++ b/testing/tests/test_hermes_cli_records_edges.py @@ -78,6 +78,17 @@ def test_small_snapshot_rejects_path_replaced_after_read( assert lanes._open_small_json_snapshot(path) is None +def test_small_snapshot_rejects_a_reader_that_exceeds_its_bound( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "record" + path.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 8) + monkeypatch.setattr(lanes, "_read_bounded", lambda *_args: b"x" * 9) + assert lanes._open_small_json_snapshot(path) is None + + def test_recovery_snapshot_classifies_nonregular_and_open_failure( tmp_path: Path, monkeypatch, diff --git a/testing/tests/test_hermes_cli_recovery_edges.py b/testing/tests/test_hermes_cli_recovery_edges.py index 27ba6773..46687826 100644 --- a/testing/tests/test_hermes_cli_recovery_edges.py +++ b/testing/tests/test_hermes_cli_recovery_edges.py @@ -87,6 +87,9 @@ def test_staged_authority_rejects_outside_nested_and_unknown_state( document["kanban_state"] = "unknown" assert lanes._staged_terminal_authority(staged, _Snapshot(document)) is None + valid = _pending_terminal_record("board", "task", 5, "accepted") + assert lanes._staged_terminal_authority(staged, _Snapshot(valid)) is None + def test_restore_staged_authority_handles_every_first_writer_state( tmp_path: Path, @@ -296,3 +299,34 @@ def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts( monkeypatch.setattr(Path, "stat", denied) assert lanes._has_pending_finalization("board", "task", 12) is False + + +def test_pending_guard_scans_invalid_prepared_and_staged_candidates( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + base = lanes.state_path("board", "task") + pending = lanes._terminal_path(base, 13) + pending.parent.mkdir(parents=True) + pending.write_text("{}", encoding="utf-8") + (pending.parent / "task.run-13.terminal.prepared-bad.json").write_text( + "{}", encoding="utf-8" + ) + (pending.parent / f"task.run-13.terminal.prepared-{'a' * 32}.json").write_text( + "{}", encoding="utf-8" + ) + staged = [ + pending.parent / f".retire.{'b' * 16}.{'c' * 16}.0", + pending.parent / f".retire.{'d' * 16}.{'e' * 16}.0", + ] + for path in staged: + path.write_text("{}", encoding="utf-8") + snapshots = iter((None, _Snapshot({}))) + monkeypatch.setattr( + lanes, + "_open_terminal_recovery_snapshot", + lambda _path: next(snapshots), + ) + monkeypatch.setattr(lanes, "_staged_terminal_authority", lambda *_args: None) + assert lanes._has_pending_finalization("board", "task", 13) is False diff --git a/testing/tests/test_hermes_cli_retention_edges.py b/testing/tests/test_hermes_cli_retention_edges.py index f7deb6a2..50f3cb19 100644 --- a/testing/tests/test_hermes_cli_retention_edges.py +++ b/testing/tests/test_hermes_cli_retention_edges.py @@ -87,3 +87,49 @@ def test_gc_skips_symlink_board_and_disappearing_candidate( monkeypatch.setattr(Path, "stat", disappear) assert lanes.gc_lane_artifacts() == 0 + + +def test_gc_counts_classified_quarantine_and_bounds_unlink_deferrals( + tmp_path: Path, + monkeypatch, +): + missing = tmp_path / "missing" + assert lanes._unlink_artifact_if_same(missing, tmp_path.stat()) is False + + root = tmp_path / "lanes" + board = root / "board" + board.mkdir(parents=True) + classified = board / "classified.candidate-1.json" + classified.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "STATE_ROOT", root) + monkeypatch.setattr( + lanes, + "_artifact_gc_candidates", + lambda _board: [classified], + ) + monkeypatch.setattr( + lanes, + "_quarantine_invalid_retained_terminal", + lambda _path: (True, True), + ) + assert lanes.gc_lane_artifacts() == 1 + + staged = board / f".retire.{'a' * 16}.{'b' * 16}.0" + staged.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + lanes, + "_artifact_gc_candidates", + lambda _board: [staged], + ) + monkeypatch.setattr( + lanes, + "_quarantine_invalid_retained_terminal", + lambda _path: (False, False), + ) + monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: None) + monkeypatch.setattr(lanes, "_unlink_artifact_if_same", lambda *_args: False) + assert lanes.gc_lane_artifacts( + max_age_seconds=-1, + max_count=0, + max_bytes=-1, + ) == 0 diff --git a/testing/tests/test_hermes_coordinator_boards.py b/testing/tests/test_hermes_coordinator_boards.py index aa63943f..c18bb7fb 100644 --- a/testing/tests/test_hermes_coordinator_boards.py +++ b/testing/tests/test_hermes_coordinator_boards.py @@ -141,7 +141,7 @@ def test_cassandra_sync_repairs_existing_worktree( assert all("GITEA_TOKEN" not in environment for environment in environments) def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" + """The credential-isolated broker needs no model-facing token to fetch.""" workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) @@ -156,8 +156,11 @@ def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch monkeypatch.setattr(coordinator.subprocess, "run", run) state = coordinator.sync_cassandra_repo({}) - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] + assert state == "ready" + assert len(commands) == 3 + assert commands[0][-3:] == ["remote", "get-url", "origin"] + assert commands[1][-4:-1] == ["remote", "set-url", "origin"] + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" diff --git a/testing/tests/test_hermes_execution_pool.py b/testing/tests/test_hermes_execution_pool.py new file mode 100644 index 00000000..8b9966d5 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool.py @@ -0,0 +1,476 @@ +"""Adversarial contracts for the fenced three-node Hermes execution pool.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services/hermes" +SCRIPTS = HERMES / "scripts" +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_client as pool_client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +KEY = b"k" * 32 + + +def binding(**values): + result = { + "board": "atlas", + "task_id": "t_deadbeef", + "run_id": "run-1234", + "worker_ordinal": 0, + "attempt": 1, + } + result.update(values) + return result + + +def assignment_payload(**values): + result = { + "context": "Implement the bounded objective.", + "assignee": "cli-auto", + "repo_url": "https://scm.bstein.dev/atlas/titan-iac.git", + "branch": "feature/hermes-safe-pool", + "base_branch": "main", + "max_runtime_seconds": 3600, + "deadline_unix": int(time.time()) + 3600, + } + result.update(values) + return result + + +def test_envelope_binds_run_ordinal_attempt_and_digest(): + signed = protocol.sign_envelope(KEY, "heartbeat", binding(), {"note": "active"}) + verified = protocol.verify_envelope(KEY, signed, expected_kind="heartbeat") + + assert verified["payload_digest"] == protocol.payload_digest({"note": "active"}) + for name, value in binding().items(): + assert verified[name] == value + + signed["attempt"] = 2 + with pytest.raises(protocol.ProtocolError, match="authentication"): + protocol.verify_envelope(KEY, signed) + + +def test_empty_poll_acknowledgement_is_authenticated(): + empty = { + "board": "", "task_id": "", "run_id": "", + "worker_ordinal": 2, "attempt": 0, + } + signed = protocol.sign_envelope(KEY, "ack", empty, {"assignment": None}) + assert protocol.verify_envelope(KEY, signed, expected_kind="ack")["payload"] == { + "assignment": None + } + + +@pytest.mark.parametrize( + "mutation,error", + [ + ({"board": "../atlas"}, "invalid board"), + ({"worker_ordinal": 3}, "outside the pool"), + ({"expires_at": 1}, "validity window"), + ], +) +def test_malformed_or_traversal_bindings_fail_closed(mutation, error): + signed = protocol.sign_envelope(KEY, "heartbeat", binding(), {"note": "active"}) + signed.update(mutation) + unsigned = dict(signed) + unsigned.pop("signature") + import hashlib + import hmac + + signed["signature"] = hmac.new( + KEY, protocol.canonical_json(unsigned), hashlib.sha256 + ).hexdigest() + with pytest.raises(protocol.ProtocolError, match=error): + protocol.verify_envelope(KEY, signed) + + +def test_oversized_and_malformed_payloads_are_rejected(): + with pytest.raises(protocol.ProtocolError, match="wire limit"): + protocol.sign_envelope( + KEY, "result", binding(), {"output": "x" * protocol.MAX_WIRE_BYTES} + ) + with pytest.raises(protocol.ProtocolError, match="malformed"): + protocol.parse_wire(b"{not-json") + with pytest.raises(protocol.ProtocolError, match="oversized"): + protocol.parse_wire(b"x" * (protocol.MAX_WIRE_BYTES + 1)) + + +def test_key_requires_private_regular_file_and_rejects_symlink(tmp_path): + key = tmp_path / "key" + key.write_bytes(KEY) + key.chmod(0o644) + with pytest.raises(protocol.ProtocolError, match="private"): + protocol.read_key(key) + key.chmod(0o600) + assert protocol.read_key(key) == KEY + link = tmp_path / "link" + link.symlink_to(key) + with pytest.raises(protocol.ProtocolError, match="unavailable"): + protocol.read_key(link) + with pytest.raises(protocol.ProtocolError, match="unavailable"): + protocol.read_key(tmp_path / "missing") + + +def test_local_signing_boundary_rejects_foreign_or_unassigned_result(monkeypatch): + monkeypatch.setattr(pool_client, "ORDINAL", 0) + boundary = pool_client.ClientBoundary(KEY) + request = {"binding": binding(), "payload": {"note": "active"}} + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary.heartbeat(request) + boundary.current = binding() + request["binding"] = binding(worker_ordinal=1) + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary.heartbeat(request) + + +def test_simultaneous_claim_materialization_has_one_winner(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + barrier = threading.Barrier(8) + outcomes = [] + + def add(index): + barrier.wait() + try: + outcome = store.add( + binding(task_id=f"t_{index}", run_id=f"run-{index}"), + assignment_payload(), + ) + except protocol.ProtocolError: + outcome = False + outcomes.append(outcome) + + threads = [threading.Thread(target=add, args=(index,)) for index in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert outcomes.count(True) == 1 + assert store.available_ordinals() == [1, 2] + + +def test_duplicate_assignment_is_idempotent_but_conflict_is_rejected(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + assert store.add(binding(), assignment_payload()) is True + assert store.add(binding(), assignment_payload()) is False + with pytest.raises(protocol.ProtocolError, match="conflicting"): + store.add(binding(), assignment_payload(context="different")) + with pytest.raises(protocol.ProtocolError, match="conflicting"): + store.add(binding(worker_ordinal=1), assignment_payload()) + + +def test_duplicate_heartbeat_and_result_are_idempotent(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + store.offer(0) + heartbeat = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "active"}, delivery_id="delivery-1" + ) + assert store.heartbeat(heartbeat) == (True, False) + assert store.heartbeat(heartbeat) == (True, True) + result = protocol.sign_envelope( + KEY, "result", binding(), {"structured": {"status": "completed"}} + ) + _, duplicate = store.accept_result(result) + assert duplicate is False + _, duplicate = store.accept_result(result) + assert duplicate is True + + +def test_reused_delivery_and_stale_attempt_cannot_cross_runs(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + first = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "one"}, delivery_id="delivery-1" + ) + store.heartbeat(first) + reused = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "two"}, delivery_id="delivery-1" + ) + with pytest.raises(protocol.ProtocolError, match="reused"): + store.heartbeat(reused) + store.finalize(binding(), "finalized") + store.add(binding(run_id="other-run"), assignment_payload()) + cross_run = protocol.sign_envelope( + KEY, "heartbeat", binding(run_id="other-run"), {"note": "one"}, + delivery_id="delivery-1", + ) + with pytest.raises(protocol.ProtocolError, match="reused"): + store.heartbeat(cross_run) + stale = protocol.sign_envelope( + KEY, "result", binding(attempt=2), {"structured": {"status": "completed"}} + ) + with pytest.raises(protocol.ProtocolError, match="attempt is stale"): + store.accept_result(stale) + replacement = protocol.sign_envelope( + KEY, + "result", + binding(run_id="replacement-run"), + {"structured": {"status": "completed"}}, + ) + with pytest.raises(protocol.ProtocolError, match="unknown or stale"): + store.accept_result(replacement) + + +def test_restart_heartbeat_loss_and_node_replacement_reuse_durable_assignment(tmp_path): + database = tmp_path / "pool.db" + first = protocol.PoolStore(database) + first.add(binding(), assignment_payload()) + offered = first.offer(0) + assert offered and offered["attempt"] == 1 + # Reopening the SQLite ledger models coordinator restart. An expired lease + # is deliberately not reassigned across ordinals; the StatefulSet replaces + # ordinal 0 with the same PVC and provider session state. + restarted = protocol.PoolStore(database) + with restarted._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=0") + resumed = restarted.offer(0) + assert resumed and resumed["run_id"] == "run-1234" + assert resumed["attempt"] == 1 + assert restarted.offer(1) is None + assert len(restarted.active_assignments()) == 1 + + +def test_result_conflict_never_overwrites_first_result(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + first = protocol.sign_envelope(KEY, "result", binding(), {"returncode": 0}) + store.accept_result(first) + conflict = protocol.sign_envelope(KEY, "result", binding(), {"returncode": 1}) + with pytest.raises(protocol.ProtocolError, match="conflicting result"): + store.accept_result(conflict) + + +def test_scm_workspace_is_ordinal_contained_and_rejects_symlink(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setattr(scm, "ORDINAL", 0) + signed = protocol.sign_envelope(KEY, "assignment", binding(), assignment_payload()) + path = scm.workspace_path(signed) + assert path == tmp_path / "runs/atlas/t_deadbeef/run-1234" + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(tmp_path / "elsewhere", target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="symlink"): + scm.workspace_path(signed) + root_case = tmp_path / "root-case" + root_case.mkdir() + root = tmp_path / "other-root" + root.mkdir() + (root_case / "runs").symlink_to(root, target_is_directory=True) + monkeypatch.setattr(scm, "WORKSPACE_ROOT", root_case) + with pytest.raises(protocol.ProtocolError, match="run root"): + scm.workspace_path(signed) + + +def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch): + monkeypatch.setattr(scm, "ORDINAL", 0) + for changes, message in ( + ({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "outside Atlas"), + ({"branch": "main"}, "reviewed namespace"), + ): + signed = protocol.sign_envelope( + KEY, "assignment", binding(), assignment_payload(**changes) + ) + with pytest.raises(protocol.ProtocolError, match=message): + scm._binding(signed) + signed = protocol.sign_envelope( + KEY, "assignment", binding(worker_ordinal=1), assignment_payload() + ) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + scm._binding(signed) + + +def test_scm_git_process_has_no_credential_or_ambient_configuration(): + environment = scm._git_environment() + assert environment == { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + assert scm._broker_repo("titan-iac").startswith("http://hermes-scm-broker.") + source = (SCRIPTS / "execution_pool_scm.py").read_text() + assert "GITEA_TOKEN" not in source and "GIT_ASKPASS" not in source + + +def test_activity_is_bounded_sanitized_and_nofollow(tmp_path): + text = coordinator.sanitize_activity( + 'route=codex token=super-secret\nAuthorization: Bearer abcdefghijklmnop\n' + '"refresh_token":"sk-ant-oat01-abcdefghijklmnopqrstuvwxyz"' + ) + assert "super-secret" not in text + assert "abcdefghijklmnop" not in text + assert "abcdefghijklmnopqrstuvwxyz" not in text + assert len(text.encode()) <= protocol.MAX_ACTIVITY_BYTES + + class FakeKanban: + @staticmethod + def worker_log_path(_task, board): + assert board == "atlas" + return str(tmp_path / "worker.log") + + envelope = {**binding(), "payload": {"activity": "visible worker activity\n"}} + coordinator._append_activity(FakeKanban, envelope) + assert (tmp_path / "worker.log").read_text() == "visible worker activity\n" + (tmp_path / "worker.log").unlink() + (tmp_path / "worker.log").symlink_to(tmp_path / "target") + with pytest.raises(protocol.ProtocolError, match="symlink"): + coordinator._append_activity(FakeKanban, envelope) + + +def _documents(path): + return [item for item in yaml.safe_load_all(path.read_text()) if item] + + +def test_three_node_statefulset_contract_and_cross_worker_isolation(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"]["spec"] + assert stateful["spec"]["replicas"] == 3 + assert stateful["spec"]["podManagementPolicy"] == "Parallel" + assert pod["automountServiceAccountToken"] is False + claims = { + item["metadata"]["name"]: item + for item in stateful["spec"]["volumeClaimTemplates"] + } + assert set(claims) == {"workspace", "provider-access"} + assert all(item["spec"]["accessModes"] == ["ReadWriteOnce"] for item in claims.values()) + anti = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + assert anti[0]["topologyKey"] == "kubernetes.io/hostname" + spread = pod["topologySpreadConstraints"][0] + assert spread["whenUnsatisfiable"] == "DoNotSchedule" + assert spread["maxSkew"] == 1 + + +def test_worker_placement_prefers_accelerators_and_preserves_exclusions(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"]["spec"] + assert pod["priorityClassName"] == "scavenger" + affinity = stateful["spec"]["template"]["spec"]["affinity"]["nodeAffinity"] + terms = affinity["requiredDuringSchedulingIgnoredDuringExecution"]["nodeSelectorTerms"] + required = {item["key"]: item for item in terms[0]["matchExpressions"]} + assert required["node-role.kubernetes.io/worker"]["values"] == ["true"] + excluded = set(required["kubernetes.io/hostname"]["values"]) + assert {"titan-04", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][0]["weight"] == 100 + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][1]["weight"] == 50 + worker_container = pod["containers"][0] + assert worker_container["resources"] == { + "requests": {"cpu": "5m", "memory": "128Mi", "ephemeral-storage": "1Gi"}, + "limits": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "8Gi"}, + } + mediators = [ + item + for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert all( + item["spec"]["template"]["spec"]["containers"][0]["resources"][ + "requests" + ] + == {"cpu": "2m", "memory": "64Mi"} + for item in mediators + ) + + +def test_model_worker_has_no_scm_or_cluster_credential_mount(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"] + containers = {item["name"]: item for item in pod["spec"]["containers"]} + worker = containers["execution-worker"] + environment = {item["name"] for item in worker["env"]} + assert not {"GITEA_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "KUBECONFIG"} & environment + mounts = {item["mountPath"] for item in worker["volumeMounts"]} + assert "/vault/secrets" not in mounts + assert "/pool-access" not in mounts + assert "/provider-access" in mounts + assert not any(item["name"] == "vault-auth-token" for item in worker["volumeMounts"]) + assert pod["metadata"]["annotations"]["vault.hashicorp.com/agent-inject-containers"] == "stage-worker-access" + assert set(containers) == {"execution-worker"} + assert "execution-pool-key" not in str(pod["metadata"]["annotations"]) + mediators = [ + item for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert len(mediators) == 3 + for mediator in mediators: + spec = mediator["spec"]["template"]["spec"] + assert spec["automountServiceAccountToken"] is False + privileged = spec["containers"][0] + assert {mount["mountPath"] for mount in privileged["volumeMounts"]} >= { + "/pool-access", "/scm-state", "/opt/scm", "/workspace" + } + assert "/provider-access" not in { + mount["mountPath"] for mount in privileged["volumeMounts"] + } + + +def test_worker_service_account_has_no_kubernetes_permissions(): + documents = _documents(HERMES / "execution-worker-rbac.yaml") + assert len(documents) == 1 + account = documents[0] + assert account["kind"] == "ServiceAccount" + assert account["automountServiceAccountToken"] is False + + +def test_coordinator_remains_single_state_owner_and_workers_do_not_mount_home(): + agent = _documents(HERMES / "agent-deployment.yaml")[0] + assert agent["kind"] == "Deployment" + assert agent["spec"]["replicas"] == 1 + assert agent["spec"]["strategy"]["type"] == "Recreate" + assert any( + item["name"] == "home" + and item["persistentVolumeClaim"]["claimName"] == "hermes-agent-home" + for item in agent["spec"]["template"]["spec"]["volumes"] + ) + worker_text = (HERMES / "execution-worker-statefulset.yaml").read_text() + assert "hermes-agent-home" not in worker_text + assert "kanban.db" not in worker_text + + +def test_worker_protocol_preserves_switchyard_fallback_and_visible_evidence(): + source = (SCRIPTS / "execution_pool_worker.py").read_text() + coordinator_source = (SCRIPTS / "execution_pool_coordinator.py").read_text() + server_source = (SCRIPTS / "execution_pool_server.py").read_text() + assert "cli_lane_runner.select_route" in source + assert 'alternate = "claude" if route.provider == "codex" else "codex"' in source + assert "codex_thread_id" in source and "claude_session_id" in source + assert "final_activity" in source + assert "worker_ordinal" in coordinator_source and "provider_sessions" in coordinator_source + assert "coordinator.reconcile" in server_source + + +def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600) + workspace = tmp_path / "runs/atlas/t_deadbeef/run-1234" + workspace.mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(workspace)], check=True) + subprocess.run(["git", "-C", str(workspace), "config", "user.email", "test@example.com"], check=True) + subprocess.run(["git", "-C", str(workspace), "config", "user.name", "Test"], check=True) + (workspace / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(workspace), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(workspace), "commit", "-qm", "initial"], check=True) + state_file = tmp_path / "session-state/atlas/t_deadbeef/run-1234.json" + state_file.parent.mkdir(parents=True) + state_file.write_text( + json.dumps({"terminal_at": time.time() - 7200, "workspace": str(workspace)}) + ) + assert worker.garbage_collect() == 1 + assert not workspace.exists() + assert not state_file.exists() diff --git a/testing/tests/test_hermes_execution_pool_assignment.py b/testing/tests/test_hermes_execution_pool_assignment.py new file mode 100644 index 00000000..d2fd91bd --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_assignment.py @@ -0,0 +1,208 @@ +"""SCM assignment tests that keep coordinator credentials out of worktrees.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from http.server import BaseHTTPRequestHandler +from types import SimpleNamespace + +import pytest +import yaml + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +HERMES = SCRIPTS.parent +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_project as project # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +KEY = b"k" * 32 + + +def test_new_task_uses_explicit_reviewed_atlas_default(monkeypatch): + monkeypatch.setattr( + project, + "resolve_project", + lambda board: ( + f"https://scm.bstein.dev/atlas/{board}.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace(id="t_deadbeef", workspace_path="", branch_name="") + + assert project.resolve_assignment("titan-iac", task) == ( + "https://scm.bstein.dev/atlas/titan-iac.git", + "wt/t_deadbeef", + "main", + ) + + +def test_legacy_local_workspace_remains_owned_by_local_lane(tmp_path): + task = SimpleNamespace( + id="t_deadbeef", workspace_path=str(tmp_path), branch_name="feature/safe" + ) + + assert project.distributed_workspace_eligible(task) is False + + +def test_local_git_environment_excludes_credential_boundary_paths(): + local = scm._git_environment() + assert "HERMES_SCM_PASSWORD_FILE" not in local + assert "GIT_ASKPASS" not in local + assert "GITEA_TOKEN" not in local + + +def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_path): + binding = { + "board": "atlas", "task_id": "t_deadbeef", "run_id": "run-1", + "worker_ordinal": 0, "attempt": 1, + } + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding, {"context": "safe"}) + ordinal_key = protocol.derive_ordinal_key(KEY, 0) + result = protocol.sign_envelope(ordinal_key, "result", binding, {"structured": {}}) + store.accept_result(result) + store.finalize(binding, "finalized") + pool = coordinator.Coordinator(KEY, store) + called = [] + pool.finalize = called.append + + assert pool.result(result)["payload"]["duplicate"] is True + assert called == [] + + +def test_provider_sessions_are_bound_to_exact_task_run(tmp_path, monkeypatch): + runtime = tmp_path / "runtime" + (runtime / "codex").mkdir(parents=True) + (runtime / "claude").mkdir(parents=True) + monkeypatch.setattr(worker, "ROOT", tmp_path / "worker") + (tmp_path / "worker/provider-state").mkdir(parents=True) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", tmp_path / "worker-data") + (tmp_path / "worker-data").mkdir() + monkeypatch.setenv("CODEX_HOME", str(runtime / "codex")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(runtime / "claude")) + assignment = {"board": "atlas", "task_id": "t_deadbeef", "run_id": "run-1"} + + worker._bind_provider_sessions(assignment) + + for path in ( + runtime / "codex/sessions", runtime / "claude/projects", + runtime / "claude/session-env", runtime / "claude/todos", + ): + assert path.is_symlink() + assert "atlas/t_deadbeef/run-1" in str(path.resolve()) + assert (tmp_path / "worker-data/home").is_symlink() + assert "atlas/t_deadbeef/run-1" in str((tmp_path / "worker-data/home").resolve()) + + +def test_internal_http_server_rejects_work_above_its_bound(monkeypatch): + server = protocol.BoundedHTTPServer( + ("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1 + ) + rejected = [] + monkeypatch.setattr(server, "shutdown_request", rejected.append) + assert server._slots.acquire(blocking=False) + try: + marker = object() + server.process_request(marker, ("127.0.0.1", 1)) + assert rejected == [marker] + finally: + server._slots.release() + server.server_close() + + +def test_additive_patch_replaces_local_lane_without_touching_base_deployment(): + patch = yaml.safe_load((HERMES / "execution-coordinator-patch.yaml").read_text()) + containers = patch["spec"]["template"]["spec"]["containers"] + local = next(item for item in containers if item["name"] == "cli-lane-runner") + pool = next(item for item in containers if item["name"] == "execution-pool-coordinator") + environment = {item["name"]: item["value"] for item in local["env"]} + assert environment == { + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY": "true", + "HERMES_CLI_LANE_CONCURRENCY": "1", + } + assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "128Mi"} + access = next(item for item in pool["volumeMounts"] if item["name"] == "runtime-access") + assert access["subPath"] == "execution-pool-key" and access["readOnly"] is True + + +def test_additive_network_policies_expose_only_worker_pool_and_switchyard_ports(): + documents = list(yaml.safe_load_all( + (HERMES / "execution-worker-networkpolicy.yaml").read_text() + )) + policies = {item["metadata"]["name"]: item for item in documents} + assert set(policies) == { + "hermes-execution-worker-isolation", + "hermes-execution-mediator-isolation", + "hermes-execution-pool-ingress", + "hermes-execution-switchyard-ingress", + *(f"hermes-execution-worker-mediator-{ordinal}" for ordinal in range(3)), + *(f"hermes-execution-mediator-worker-{ordinal}" for ordinal in range(3)), + } + assert policies["hermes-execution-pool-ingress"]["spec"]["ingress"][0]["ports"] == [ + {"protocol": "TCP", "port": 9007} + ] + pool_source = policies["hermes-execution-pool-ingress"]["spec"]["ingress"] + assert "hermes-execution-mediator" in str(pool_source) + worker_egress = policies["hermes-execution-worker-isolation"]["spec"]["egress"] + assert "hermes-scm-broker" not in str(worker_egress) + assert "hermes-execution-mediator" not in str(worker_egress) + for ordinal in range(3): + worker_policy = policies[f"hermes-execution-worker-mediator-{ordinal}"] + mediator_policy = policies[f"hermes-execution-mediator-worker-{ordinal}"] + assert worker_policy["spec"]["podSelector"]["matchLabels"][ + "apps.kubernetes.io/pod-index" + ] == str(ordinal) + assert worker_policy["spec"]["egress"][0]["to"][0]["podSelector"][ + "matchLabels" + ]["pool-ordinal"] == str(ordinal) + assert mediator_policy["spec"]["ingress"][0]["from"][0]["podSelector"][ + "matchLabels" + ]["apps.kubernetes.io/pod-index"] == str(ordinal) + + +@pytest.mark.parametrize( + "branch", + [ + "../main", + "main", + "feature/../../main", + ], +) +def test_assignment_rejects_unreviewed_task_branch(monkeypatch, branch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/titan-iac.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace( + id="t_deadbeef", workspace_path="", branch_name=branch, + repo_url="https://evil.example/atlas/other.git", base_branch="../main", + ) + + with pytest.raises(project.ProjectPolicyError): + project.resolve_assignment("titan-iac", task) + + +def test_registry_authority_ignores_unreviewed_task_repo_metadata(monkeypatch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/metis.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace( + id="t_deadbeef", branch_name="review/t_deadbeef", + repo_url="https://evil.example/atlas/other.git", base_branch="../main", + ) + assert project.resolve_assignment("metis", task)[:2] == ( + "https://scm.bstein.dev/atlas/metis.git", "review/t_deadbeef" + ) diff --git a/testing/tests/test_hermes_execution_pool_coordinator_v2.py b/testing/tests/test_hermes_execution_pool_coordinator_v2.py new file mode 100644 index 00000000..fa679b0d --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_coordinator_v2.py @@ -0,0 +1,293 @@ +"""Exact-run coordinator finalization, lease, and ownership contracts.""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"k" * 32 +STRUCTURED = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def assignment_payload(**changes): + value = { + "context": "objective", + "assignee": "cli-auto", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + "max_runtime_seconds": 3600, + "deadline_unix": 10_000_000_000, + } + value.update(changes) + return value + + +def signed(kind, exact_binding, payload): + key = protocol.derive_ordinal_key(MASTER, exact_binding["worker_ordinal"]) + return protocol.sign_envelope(key, kind, exact_binding, payload) + + +class Scope: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +class Connection: + def close(self): + return None + + +def install_kanban(monkeypatch, tasks, tmp_path, boards=None): + module = types.ModuleType("kanban_db") + module.tasks = {task.id: task for task in tasks} + module.completed = [] + module.blocked = [] + module.heartbeats = [] + module.reclaimed = [] + module.branches = [] + module.scoped_current_board = lambda _board: Scope() + module.connect = lambda board=None: Connection() + module.get_task = lambda _connection, task_id: module.tasks.get(task_id) + module.list_tasks = lambda _connection: list(module.tasks.values()) + module.list_boards = lambda include_archived=False: boards or ["metis"] + module.build_worker_context = lambda _connection, task_id: f"objective {task_id}" + module.worker_log_path = lambda task_id, board=None: str( + tmp_path / board / f"{task_id}.log" + ) + + def complete(_connection, task_id, **values): + module.completed.append((task_id, values)) + module.tasks[task_id].status = "completed" + return True + + def block(_connection, task_id, **values): + module.blocked.append((task_id, values)) + module.tasks[task_id].status = "blocked" + return True + + def heartbeat(_connection, task_id, **values): + module.heartbeats.append((task_id, values)) + return True + + module.complete_task = complete + module.block_task = block + module.heartbeat_worker = heartbeat + module.reclaim_task = lambda _c, task_id, **values: module.reclaimed.append( + (task_id, values) + ) + package = types.ModuleType("hermes_cli") + package.kanban_db = module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", module) + return module + + +def task(**changes): + value = { + "id": "t_deadbeef", + "status": "running", + "current_run_id": 23, + "assignee": "cli-auto", + "workspace_path": "", + "branch_name": "wt/t_deadbeef", + "max_runtime_seconds": 3600, + "goal_mode": False, + "goal_max_turns": 1, + } + value.update(changes) + return SimpleNamespace(**value) + + +def store_with_assignment(tmp_path, exact_binding=None): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(exact_binding or binding(), assignment_payload()) + return store + + +def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch): + monkeypatch.setattr( + coordinator, + "resolve_assignment", + lambda board, _task: ( + f"https://scm.bstein.dev/atlas/{board}.git", "wt/t_deadbeef", "main" + ), + ) + kanban = SimpleNamespace( + build_worker_context=lambda _connection, _task_id: {"safe": True} + ) + item = task(max_runtime_seconds=1, goal_max_turns=99) + value = coordinator.assignment_payload(kanban, object(), item, "metis") + assert json.loads(value["context"]) == {"safe": True} + assert value["max_runtime_seconds"] == 60 + assert value["goal_max_turns"] == 12 + assert value["repo_url"].endswith("/metis.git") + kanban.build_worker_context = lambda *_a: "x" * (32 * 1024 + 1) + with pytest.raises(RuntimeError, match="32KiB"): + coordinator.assignment_payload(kanban, object(), item, "metis") + + +def test_poll_uses_per_ordinal_key_for_empty_and_active_assignment(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + poll_binding = binding(board="", task_id="", run_id="", attempt=0) + request = signed("poll", poll_binding, {"ready": True}) + empty = pool.poll(request) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), empty, expected_kind="ack" + )["payload"] == {"assignment": None} + store.add(binding(), assignment_payload()) + offered = protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), pool.poll(request), + expected_kind="assignment", + ) + assert offered["run_id"] == "23" + wrong_key = protocol.derive_ordinal_key(MASTER, 1) + with pytest.raises(protocol.ProtocolError, match="authentication"): + pool.poll(protocol.sign_envelope(wrong_key, "poll", poll_binding, {})) + + +def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute( + tmp_path, monkeypatch +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + envelope = signed( + "result", + binding(), + { + "structured": dict(STRUCTURED), + "returncode": 0, + "capacity_failure": False, + "node": "titan-05", + "route": {}, + "provider_sessions": {}, + "final_activity": "done\n", + }, + ) + ack = pool.result(envelope) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), ack, expected_kind="ack" + )["payload"] == {"accepted": True, "duplicate": False} + assert kanban.completed[0][1]["expected_run_id"] == 23 + row = store._connect().execute("SELECT state FROM assignments").fetchone() + assert row[0] == "finalized" and store.available_ordinals() == [0, 1, 2] + pool.reconcile() + assert store.active_assignments() == [] + + +def test_finalize_fences_stale_run_and_blocks_failed_result_exactly(tmp_path, monkeypatch): + stale_task = task(current_run_id=24) + kanban = install_kanban(monkeypatch, [stale_task], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + pool.result(signed("result", binding(), {"structured": dict(STRUCTURED), "returncode": 0})) + assert not kanban.completed + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + second = binding(run_id="25", worker_ordinal=1) + failed_task = task(current_run_id=25) + kanban.tasks[failed_task.id] = failed_task + store.add(second, assignment_payload()) + blocked = {**STRUCTURED, "status": "blocked", "blockers": ["capacity"]} + pool.result( + signed( + "result", second, + {"structured": blocked, "returncode": 1, "capacity_failure": True}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 25 + assert kanban.blocked[-1][1]["kind"] == "transient" + + +def test_finalize_rejects_invalid_payload_and_noncanonical_run(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + with pytest.raises(protocol.ProtocolError, match="payload"): + pool.finalize({**binding(), "result": []}) + bad = binding(run_id="run-bad", worker_ordinal=1) + store.add(bad, assignment_payload()) + pool.finalize({**bad, "result": {"structured": {}}}) + states = dict(store._connect().execute("SELECT run_id,state FROM assignments")) + assert states["run-bad"] == "stale" + + +def test_heartbeat_uses_integer_exact_api_and_surfaces_activity(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = signed( + "heartbeat", binding(), + {"note": "active", "activity": "safe output\n"}, + ) + ack = pool.heartbeat(request) + assert kanban.heartbeats[0][1]["expected_run_id"] == 23 + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "safe output\n" + assert ack["kind"] == "ack" + kanban.heartbeat_worker = lambda *_a, **_k: False + with pytest.raises(protocol.ProtocolError, match="no longer owns"): + pool.heartbeat(signed("heartbeat", binding(), {})) + + +def test_recovery_defers_io_errors_and_lease_failure_releases_ordinal( + tmp_path, monkeypatch, capsys +): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr(store, "pending_results", lambda: [binding()]) + monkeypatch.setattr(pool, "finalize", lambda _record: (_ for _ in ()).throw(OSError("busy"))) + pool.recover_results() + assert "recovery deferred" in capsys.readouterr().err + + monkeypatch.undo() + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "leases.db") + exact = binding(attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + pool = coordinator.Coordinator(MASTER, store) + pool.expire_leases() + assert kanban.blocked[0][1]["expected_run_id"] == 23 + assert kanban.blocked[0][1]["kind"] == "transient" + assert store.available_ordinals() == [0, 1, 2] diff --git a/testing/tests/test_hermes_execution_pool_dispatch_v2.py b/testing/tests/test_hermes_execution_pool_dispatch_v2.py new file mode 100644 index 00000000..57dd0f74 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_dispatch_v2.py @@ -0,0 +1,410 @@ +"""Coordinator dispatch migration and versioned server contracts.""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_server as server # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + MASTER, + assignment_payload, + binding, + install_kanban, + store_with_assignment, + task, +) +from testing.tests.test_hermes_execution_pool_mediator import http_request # noqa: E402 + + +def test_reconcile_fences_old_run_then_recovers_current_pathless_run( + tmp_path, monkeypatch +): + item = task(current_run_id=24) + install_kanban(monkeypatch, [item], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + pool.reconcile() + rows = store._connect().execute( + "SELECT run_id,state,worker_ordinal FROM assignments ORDER BY run_id" + ).fetchall() + assert [tuple(row) for row in rows] == [ + ("23", "stale", 0), ("24", "assigned", 0) + ] + + +def test_reconcile_preserves_owned_workspace_and_exactly_blocks_registry_failure( + tmp_path, monkeypatch +): + owned = task(id="t_owned", current_run_id=30, workspace_path="/owned") + broken = task(id="t_broken", current_run_id=31) + kanban = install_kanban(monkeypatch, [owned, broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + + def prepare(_db, _connection, item, _board): + if item.id == "t_broken": + raise RuntimeError("registry unavailable") + return assignment_payload() + + monkeypatch.setattr(coordinator, "assignment_payload", prepare) + pool.reconcile() + assert store.active_assignments() == [] + assert kanban.blocked[0][0] == "t_broken" + assert kanban.blocked[0][1]["expected_run_id"] == 31 + assert all(call[0] != "t_owned" for call in kanban.blocked) + + +def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch( + tmp_path, monkeypatch +): + item = task(status="running") + install_kanban(monkeypatch, [item], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + observed = [] + + def claim(active, limit, eligible): + observed.append((active, limit, eligible("metis", item))) + return [("metis", item.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", claim) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_a: assignment_payload(branch="wt/t_deadbeef"), + ) + pool.dispatch() + assert observed == [(set(), 3, True)] + record = store.active_assignments()[0] + assert record["run_id"] == "23" + assert record["payload"]["branch"] == "wt/t_deadbeef" + + +def test_dispatch_incompatible_unversioned_claim_api_fails_closed( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + calls = [] + + def old_claim(active, limit): + calls.append((active, limit)) + return [("metis", owned.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", old_claim) + with pytest.raises(TypeError): + pool.dispatch() + assert calls == [] + assert store.active_assignments() == [] + assert kanban.reclaimed == [] + + +def test_dispatch_workspace_ownership_race_is_exactly_fenced( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", owned.id)], + ) + pool.dispatch() + assert store.active_assignments() == [] + assert kanban.blocked == [ + ( + owned.id, + { + "reason": ( + "Distributed claim fenced because an existing workspace " + "is owned by the local lane" + ), + "kind": "capability", + "expected_run_id": 23, + }, + ) + ] + + +def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim( + tmp_path, monkeypatch +): + broken = task(current_run_id=None) + kanban = install_kanban(monkeypatch, [broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", broken.id)], + ) + pool.dispatch() + assert kanban.blocked[0][1]["expected_run_id"] is None + assert "canonical run ID" in kanban.blocked[0][1]["reason"] + + full = protocol.PoolStore(tmp_path / "full.db") + for ordinal in range(3): + full.add( + binding(task_id=f"t_{ordinal}", run_id=str(ordinal + 1), worker_ordinal=ordinal), + assignment_payload(), + ) + pool = coordinator.Coordinator(MASTER, full) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: (_ for _ in ()).throw(AssertionError("must not claim")), + ) + pool.dispatch() + + +def test_lease_expiry_with_noncanonical_run_is_released_as_stale(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + exact = binding(run_id="bad-run", attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + +def test_terminal_activity_marker_prevents_duplicate_append(tmp_path): + class Kanban: + worker_log_path = staticmethod( + lambda task_id, board=None: str(tmp_path / f"{board}-{task_id}.log") + ) + + record = { + **binding(), + "result_digest": "a" * 64, + "result": {"final_activity": "terminal evidence\n"}, + } + coordinator._append_terminal_activity(Kanban, record) + coordinator._append_terminal_activity(Kanban, record) + text = (tmp_path / "metis-t_deadbeef.log").read_text() + assert text.count("execution-pool-result") == 1 + empty = {**record, "result": {"final_activity": ""}} + coordinator._append_terminal_activity(Kanban, empty) + + +def test_activity_and_terminal_log_reject_invalid_or_symlink_targets(tmp_path): + class Kanban: + worker_log_path = staticmethod(lambda *_a, **_k: str(tmp_path / "worker.log")) + + with pytest.raises(protocol.ProtocolError, match="payload"): + coordinator._append_activity(Kanban, {**binding(), "payload": []}) + coordinator._append_activity(Kanban, {**binding(), "payload": {"activity": ""}}) + (tmp_path / "worker.log").symlink_to(tmp_path / "target") + with pytest.raises(protocol.ProtocolError, match="symlink"): + coordinator._append_terminal_activity( + Kanban, + { + **binding(), "result_digest": "a" * 64, + "result": {"final_activity": "terminal"}, + }, + ) + + +def test_heartbeat_bad_run_duplicate_and_unstructured_result_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path) + bad_binding = binding(run_id="bad-run") + bad_store = protocol.PoolStore(tmp_path / "bad.db") + bad_store.add(bad_binding, assignment_payload()) + bad_store.offer(0) + pool = coordinator.Coordinator(MASTER, bad_store) + key = protocol.derive_ordinal_key(MASTER, 0) + with pytest.raises(protocol.ProtocolError, match="canonical"): + pool.heartbeat(protocol.sign_envelope(key, "heartbeat", bad_binding, {})) + + store = store_with_assignment(tmp_path / "duplicate") + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = protocol.sign_envelope( + key, "heartbeat", binding(), {"activity": "once"}, + delivery_id="same-heartbeat", + ) + pool.heartbeat(request) + pool.heartbeat(request) + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "once" + + second = binding(run_id="24", worker_ordinal=1) + item.current_run_id = 24 + store.add(second, assignment_payload()) + pool.result( + protocol.sign_envelope( + protocol.derive_ordinal_key(MASTER, 1), "result", second, + {"structured": [], "returncode": 1}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 24 + + +def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path, boards=["", "metis"]) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + original_connect = kanban.connect + kanban.list_boards = lambda include_archived=False: [] + kanban.connect = lambda board=None: (_ for _ in ()).throw(OSError("busy")) + pool.reconcile() + kanban.connect = original_connect + kanban.list_boards = lambda include_archived=False: [""] + pool.reconcile() + + full = SimpleNamespace( + active_assignments=lambda: [], available_ordinals=lambda: [] + ) + coordinator.Coordinator(MASTER, full).reconcile() + + empty = protocol.PoolStore(tmp_path / "missing.db") + monkeypatch.setattr( + coordinator.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")] + ) + coordinator.Coordinator(MASTER, empty).dispatch() + + +def test_nonterminal_lease_expiry_is_reoffered_without_kanban_block(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert kanban.blocked == [] + assert store.active_assignments()[0]["attempt"] == 2 + + +class HTTPStore: + def __init__(self, fail=False): + self.fail = fail + + def available_ordinals(self): + if self.fail: + raise sqlite3.Error("busy") + return [0, 1, 2] + + +class HTTPCoordinator: + def __init__(self, fail_ready=False): + self.store = HTTPStore(fail_ready) + + @staticmethod + def poll(value): + return {"route": "poll", "value": value} + + @staticmethod + def heartbeat(_value): + raise protocol.ProtocolError("stale") + + @staticmethod + def result(_value): + raise RuntimeError("storage") + + +def test_versioned_server_routes_readiness_and_errors(): + handler = server.handler_factory(HTTPCoordinator()) + assert http_request(handler, "/ready") == (200, {"ready": True, "version": 2}) + assert http_request(handler, "/missing")[0] == 404 + body = protocol.canonical_json({"safe": True}) + assert http_request(handler, "/v1/poll", body=body) == ( + 200, {"route": "poll", "value": {"safe": True}} + ) + assert http_request(handler, "/v1/heartbeat", body=body)[0] == 409 + assert http_request(handler, "/v1/result", body=body)[0] == 503 + assert http_request(handler, "/unknown", body=body)[0] == 404 + assert http_request(handler, "/v1/poll", body=b"")[0] == 409 + assert http_request(server.handler_factory(HTTPCoordinator(True)), "/ready")[0] == 503 + + +class RunCoordinator: + instances = [] + + def __init__(self, key, store): + self.key = key + self.store = store + self.calls = [] + self.dispatch_count = 0 + self.__class__.instances.append(self) + + def expire_leases(self): + self.calls.append("expire") + + def recover_results(self): + self.calls.append("recover") + + def reconcile(self): + self.calls.append("reconcile") + + def dispatch(self): + self.calls.append("dispatch") + self.dispatch_count += 1 + if self.dispatch_count > 1: + raise RuntimeError("deferred") + + +def test_server_once_and_maintenance_loop_are_versioned_and_resilient( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr(server, "STATE_ROOT", tmp_path) + monkeypatch.setattr(server, "read_key", lambda _path: MASTER) + monkeypatch.setattr(sys, "argv", ["pool", "--once"]) + assert server.run(RunCoordinator) == 0 + assert RunCoordinator.instances[-1].calls == [ + "expire", "recover", "reconcile", "dispatch" + ] + + started = [] + + class FakeServer: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(sys, "argv", ["pool"]) + monkeypatch.setattr(server, "BoundedHTTPServer", FakeServer) + monkeypatch.setattr( + server.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(StopIteration()), + ) + with pytest.raises(StopIteration): + server.run(RunCoordinator) + assert started == [(("0.0.0.0", server.PORT), 8)] + assert "maintenance deferred" in capsys.readouterr().err + + +def test_coordinator_compatibility_exports_delegate_to_server(monkeypatch): + marker = object() + monkeypatch.setattr(server, "handler_factory", lambda value: (marker, value)) + assert coordinator.handler_factory(marker) == (marker, marker) + monkeypatch.setattr(server, "run", lambda value: 17 if value is coordinator.Coordinator else 0) + assert coordinator.main() == 17 diff --git a/testing/tests/test_hermes_execution_pool_mediator.py b/testing/tests/test_hermes_execution_pool_mediator.py new file mode 100644 index 00000000..e5c6c996 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_mediator.py @@ -0,0 +1,447 @@ +"""Isolated mediator, SCM gate, and model-facing API contracts.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_client as client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 + + +KEY = b"d" * 64 +RESULT = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "42", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def payload(**changes): + value = { + "context": "safe objective", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + } + value.update(changes) + return value + + +def assignment(**payload_changes): + return protocol.sign_envelope( + KEY, "assignment", binding(), payload(**payload_changes) + ) + + +class Response: + def __init__(self, body): + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.body + + +class FakeSCM: + def __init__(self): + self.submitted = [] + + def checkout(self, _envelope): + return {"workspace": "/workspace/run", "baseline_sha": "a" * 40} + + def submit(self, envelope, request): + self.submitted.append((envelope, request)) + return {"pull_request": "https://scm.bstein.dev/atlas/metis/pulls/7"} + + +def test_result_schema_validation_rejects_every_unsafe_shape(): + invalid = [ + None, + {}, + {"structured": []}, + {"structured": {**RESULT, "extra": True}}, + {"structured": {**RESULT, "status": "unknown"}}, + {"structured": {**RESULT, "summary": ""}}, + {"structured": {**RESULT, "tests_run": "pytest"}}, + {"structured": {**RESULT, "tests_run": [7]}}, + ] + for value in invalid: + with pytest.raises(protocol.ProtocolError): + client._validate_result(value) + assert client._validate_result({"structured": dict(RESULT)})["structured"] == RESULT + + +def test_client_post_verifies_response_and_bounds_body(monkeypatch): + boundary = client.ClientBoundary(KEY, FakeSCM()) + ack = protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True}) + monkeypatch.setattr(client.urllib.request, "urlopen", lambda *_a, **_k: Response(protocol.canonical_json(ack))) + assert boundary._post("/v1/result", ack)["kind"] == "ack" + monkeypatch.setattr( + client.urllib.request, + "urlopen", + lambda *_a, **_k: Response(b"x" * (protocol.MAX_WIRE_BYTES + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="exceeds"): + boundary._post("/v1/result", ack) + + +def test_poll_materializes_only_ordinal_owned_assignment(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", 0) + boundary = client.ClientBoundary(KEY, FakeSCM()) + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, + "ack", + binding(board="", task_id="", run_id="", attempt=0), + {"assignment": None}, + ) + assert boundary.poll() == {"assignment": None} + + boundary._post = lambda *_a: assignment() + result = boundary.poll()["assignment"] + assert result["workspace"] == "/workspace/run" + assert result["protocol_version"] == 2 + assert "signature" not in result + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "assignment", binding(worker_ordinal=1), payload() + ) + with pytest.raises(protocol.ProtocolError, match="foreign"): + boundary.poll() + + +def test_heartbeat_and_finish_require_exact_current_binding(monkeypatch): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + current = assignment() + boundary.current = current + + def post(path, envelope): + kind = "heartbeat" if path.endswith("heartbeat") else "result" + protocol.verify_envelope(KEY, envelope, expected_kind=kind) + return protocol.sign_envelope( + KEY, "ack", binding(), {"accepted": True, "duplicate": False} + ) + + boundary._post = post + assert boundary.heartbeat( + {"binding": binding(), "payload": {"note": "active"}} + )["ack"]["accepted"] + request = { + "binding": binding(), + "payload": {"structured": dict(RESULT), "returncode": 0}, + "title": "Safe change", + "body": "Evidence", + } + finished = boundary.finish(request) + assert finished["ack"]["accepted"] and boundary.current is None + assert fake_scm.submitted + assert finished["structured"]["artifacts"] == [ + "https://scm.bstein.dev/atlas/metis/pulls/7" + ] + + boundary.current = current + with pytest.raises(protocol.ProtocolError, match="heartbeat payload"): + boundary.heartbeat({"binding": binding(), "payload": "bad"}) + with pytest.raises(protocol.ProtocolError, match="object"): + boundary._current_for(None) + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary._current_for(binding(attempt=2)) + + +def test_noncompleted_finish_never_invokes_scm_and_bad_ack_is_rejected(): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + boundary.current = assignment() + bad = protocol.sign_envelope(KEY, "ack", binding(attempt=2), {"accepted": True}) + boundary._post = lambda *_a: bad + blocked = {**RESULT, "status": "blocked", "blockers": ["capacity"]} + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.finish( + {"binding": binding(), "payload": {"structured": blocked, "returncode": 1}} + ) + assert fake_scm.submitted == [] + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "result", binding(), {"accepted": True} + ) + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.heartbeat({"binding": binding(), "payload": {}}) + + +def http_request(handler, path, *, body=None): + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), handler, max_workers=2) + thread = threading.Thread(target=server.handle_request) + thread.start() + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{path}", + data=body, + method="POST" if body is not None else "GET", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=3) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + finally: + thread.join(timeout=3) + server.server_close() + + +def test_model_api_exposes_only_gated_state_machine_operations(): + class Boundary: + poll = staticmethod(lambda: {"assignment": None}) + heartbeat = staticmethod(lambda request: {"heartbeat": request["payload"]}) + finish = staticmethod(lambda request: {"finish": request["payload"]}) + + handler = client.handler_factory(Boundary()) + assert http_request(handler, "/ready") == ( + 200, {"protocol_version": 2, "ready": True} + ) + assert http_request(handler, "/missing")[0] == 404 + for operation in ("poll", "heartbeat", "finish"): + body = protocol.canonical_json( + {"operation": operation, "binding": binding(), "payload": {}} + ) + assert http_request(handler, "/v1/client", body=body)[0] == 200 + for value in ( + {"operation": "bypass"}, + {"operation": "poll", "authority": "steal"}, + ): + status, response = http_request( + handler, "/v1/client", body=protocol.canonical_json(value) + ) + assert status == 409 and response.get("error") + + +def test_client_main_validates_ordinal_and_starts_bounded_server(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", -1) + with pytest.raises(SystemExit, match="ORDINAL"): + client.main() + started = [] + + class Server: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(client, "ORDINAL", 2) + monkeypatch.setattr(client, "read_key", lambda _path: KEY) + monkeypatch.setattr(client, "BoundedHTTPServer", Server) + monkeypatch.setattr(client, "SCMBoundary", lambda _key: FakeSCM()) + assert client.main() == 0 + assert started == [(("0.0.0.0", client.PORT), 4)] + + +def init_checkout(path, branch="wt/t_deadbeef"): + subprocess.run(["git", "init", "-q", str(path)], check=True) + commands = [ + ("remote", "add", "origin", "https://scm.bstein.dev/atlas/metis.git"), + ( + "remote", + "add", + "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ), + ("checkout", "-qb", branch), + ("config", "user.email", "test@example.com"), + ("config", "user.name", "Test"), + ] + for command in commands: + subprocess.run(["git", "-C", str(path), *command], check=True) + (path / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + return subprocess.check_output( + ["git", "-C", str(path), "rev-parse", "HEAD"], text=True + ).strip() + + +def test_scm_run_and_binding_validation(tmp_path, monkeypatch): + checkout = tmp_path / "checkout" + head = init_checkout(checkout) + assert scm._run("rev-parse", "HEAD", cwd=checkout) == head + with pytest.raises(RuntimeError): + scm._run("rev-parse", "missing", cwd=checkout) + monkeypatch.setattr(scm, "MAX_STATUS_BYTES", 1) + with pytest.raises(protocol.ProtocolError, match="output"): + scm._run("rev-parse", "HEAD", cwd=checkout) + + monkeypatch.setattr(scm, "ORDINAL", 0) + assert scm._binding(assignment())[1:] == ("metis", "wt/t_deadbeef", "main") + for envelope in ( + {**assignment(), "kind": "result"}, + {**assignment(), "payload": []}, + assignment(repo_url="https://evil.example/metis.git"), + assignment(branch="main"), + assignment(base_branch="../main"), + protocol.sign_envelope(KEY, "assignment", binding(worker_ordinal=1), payload()), + ): + with pytest.raises(protocol.ProtocolError): + scm._binding(envelope) + + +def test_scm_private_paths_and_text_are_bounded(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + target = scm.workspace_path(assignment()) + assert target.name == "42" + assert scm._state_path(assignment()).name == "metis-t_deadbeef-42.json" + bad = {**assignment(), "task_id": "../bad"} + with pytest.raises(protocol.ProtocolError, match="binding"): + scm.workspace_path(bad) + state = tmp_path / "regular" + state.write_text("safe") + assert scm._regular_text(state, 10) == "safe" + with pytest.raises(protocol.ProtocolError, match="invalid"): + scm._regular_text(state, 1) + binary = tmp_path / "binary" + binary.write_bytes(b"\xff") + with pytest.raises(protocol.ProtocolError, match="malformed"): + scm._regular_text(binary, 10) + linked_state = tmp_path / "linked-state" + linked_state.mkdir() + (tmp_path / "state-link").symlink_to(linked_state, target_is_directory=True) + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state-link") + with pytest.raises(protocol.ProtocolError): + scm._state_path(assignment()) + + +def test_boundary_verification_existing_checkout_and_missing_baseline(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + state_root = tmp_path / "state" + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + monkeypatch.setattr(scm, "SCM_ROOT", state_root) + monkeypatch.setattr(scm, "ORDINAL", 0) + boundary = scm.Boundary(KEY) + assert boundary.verify(assignment())["kind"] == "assignment" + destination = scm.workspace_path(assignment()) + baseline = init_checkout(destination) + protocol.atomic_json( + scm._state_path(assignment()), + {"baseline_sha": baseline, "repo": "metis", "branch": "wt/t_deadbeef"}, + ) + assert boundary.checkout(assignment())["baseline_sha"] == baseline + scm._state_path(assignment()).write_text("{}") + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.checkout(assignment()) + + +@pytest.mark.parametrize("feature_exists", [True, False]) +def test_new_checkout_uses_broker_and_safe_base_fallback( + tmp_path, monkeypatch, feature_exists +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + calls = [] + + def run(*arguments, cwd=None, timeout=300): + calls.append((arguments, cwd, timeout)) + if arguments[0] == "clone": + branch = arguments[arguments.index("--branch") + 1] + destination = Path(arguments[-1]) + if branch == "wt/t_deadbeef" and not feature_exists: + raise RuntimeError("missing branch") + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + result = scm.Boundary(KEY).checkout(assignment()) + assert result["baseline_sha"] == "a" * 40 + assert any("hermes-scm-broker" in str(call) for call in calls) + if not feature_exists: + assert any(call[0][0] == "checkout" for call in calls) + + +def test_failed_clone_never_deletes_unmanaged_state(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + + def fail(*arguments, **_kwargs): + destination = Path(arguments[-1]) + destination.mkdir(parents=True, exist_ok=True) + (destination / "preserved").write_text("owner data") + raise RuntimeError("clone failed") + + monkeypatch.setattr(scm, "_run", fail) + with pytest.raises(protocol.ProtocolError, match="unmanaged"): + scm.Boundary(KEY).checkout(assignment()) + assert (scm.workspace_path(assignment()) / "preserved").read_text() == "owner data" + + +def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch): + existing = json.dumps([{"html_url": "https://scm/pulls/1"}]).encode() + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: existing) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/1" + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]") + monkeypatch.setattr( + scm.scm_broker_client, "create_draft", + lambda *_a, **_k: json.dumps({"html_url": "https://scm/pulls/2"}).encode(), + ) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/2" + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + state = scm._state_path(assignment()) + protocol.atomic_json(state, {"baseline_sha": "a" * 40}) + boundary = scm.Boundary(KEY) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "b" * 40) + outputs = {"status": "", "rev-list": "1", "push": ""} + + def run(*arguments, **_kwargs): + return outputs.get(arguments[0], "") + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(boundary, "_draft", lambda *_a: "https://scm/pulls/3") + result = boundary.submit(assignment(), {"title": "safe", "body": "evidence"}) + assert result["pull_request"] == "https://scm/pulls/3" + outputs["status"] = "?? untracked" + with pytest.raises(protocol.ProtocolError, match="uncommitted"): + boundary.submit(assignment(), {}) + outputs["status"] = "" + outputs["rev-list"] = "0" + assert boundary.submit(assignment(), {})["pull_request"] == "" + with pytest.raises(protocol.ProtocolError, match="metadata"): + boundary.submit(assignment(), {"title": "x" * 513, "body": "x"}) diff --git a/testing/tests/test_hermes_execution_pool_project.py b/testing/tests/test_hermes_execution_pool_project.py new file mode 100644 index 00000000..16c57427 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_project.py @@ -0,0 +1,211 @@ +"""Canonical board-registry and Git-ref assignment contracts.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_project as project # noqa: E402 + + +def git(*arguments: str, cwd: Path | None = None) -> str: + return subprocess.check_output( + ["git", *arguments], cwd=cwd, text=True, stderr=subprocess.DEVNULL + ).strip() + + +def registry(tmp_path, monkeypatch, board="metis", remote=None, base=None): + projects = tmp_path / "projects" + boards = tmp_path / "boards" + checkout = projects / board + checkout.mkdir(parents=True) + git("init", "-q", cwd=checkout) + git( + "remote", "add", "origin", + remote or f"https://scm.bstein.dev/atlas/{board}.git", + cwd=checkout, + ) + if base: + git( + "symbolic-ref", "refs/remotes/origin/HEAD", f"refs/remotes/origin/{base}", + cwd=checkout, + ) + entry = boards / board + entry.mkdir(parents=True) + (entry / "board.json").write_text( + json.dumps({"slug": board, "default_workdir": str(checkout)}) + ) + monkeypatch.setattr(project, "PROJECT_ROOT", projects) + monkeypatch.setattr(project, "BOARD_ROOT", boards) + return checkout, entry / "board.json" + + +@pytest.mark.parametrize( + "branch", + [ + "feature/safe", "fix/safe", "chore/safe", "docs/safe", "test/safe", + "refactor/safe", "wt/t_deadbeef", "review/t_deadbeef", "hermes/safe", + "handoff/safe", + ], +) +def test_every_established_feature_namespace_is_accepted(branch): + assert project.validate_branch(branch, feature=True) == branch + + +@pytest.mark.parametrize( + "branch", + ["main", "unknown/safe", "../main", "feature/../../main", "feature/.hidden", "-bad", "é/safe", "x" * 201], +) +def test_ref_validation_denies_traversal_and_unreviewed_namespaces(branch): + with pytest.raises(project.ProjectPolicyError): + project.validate_branch(branch, feature=True) + + +def test_canonical_registry_resolves_repo_base_and_default_branch(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch, base="trunk") + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", "trunk", checkout.resolve() + ) + task = SimpleNamespace(id="t_deadbeef", branch_name="") + assert project.resolve_assignment("metis", task) == ( + "https://scm.bstein.dev/atlas/metis.git", "wt/t_deadbeef", "trunk" + ) + + +@pytest.mark.parametrize("board", ["cassandra", "metis", "soteria", "titan-iac"]) +def test_every_atlas_project_uses_its_own_registry_checkout(tmp_path, monkeypatch, board): + registry(tmp_path, monkeypatch, board=board) + repo, base, _ = project.resolve_project(board) + assert repo == f"https://scm.bstein.dev/atlas/{board}.git" + assert base == "main" + + +def test_registered_project_without_checkout_keeps_its_own_repo_identity( + tmp_path, monkeypatch +): + checkout, _ = registry(tmp_path, monkeypatch, board="metis") + shutil.rmtree(checkout) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_project_slug_too_long_for_an_atlas_repo( + tmp_path, monkeypatch +): + board = "m" * 101 + checkout, _ = registry(tmp_path, monkeypatch, board=board) + shutil.rmtree(checkout) + with pytest.raises(project.ProjectPolicyError, match="repository identity"): + project.resolve_project(board) + + +def test_noncanonical_remote_head_falls_back_to_main(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + real_run = project._run_git + + def run_git(workdir, *arguments): + if arguments[0] == "symbolic-ref": + return "heads/not-origin" + return real_run(workdir, *arguments) + + monkeypatch.setattr(project, "_run_git", run_git) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_slug_traversal_symlink_and_malformed_documents(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="slug"): + project._read_board("../metis") + board_file.write_text("not-json") + with pytest.raises(project.ProjectPolicyError, match="malformed"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "other", "default_workdir": "/tmp"})) + with pytest.raises(project.ProjectPolicyError, match="identity"): + project._read_board("metis") + board_file.unlink() + board_file.symlink_to("/etc/passwd") + with pytest.raises(OSError): + project._read_board("metis") + + +def test_registry_rejects_oversized_archived_and_non_regular_entry(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + board_file.write_text("x" * (project.MAX_BOARD_BYTES + 1)) + with pytest.raises(project.ProjectPolicyError, match="bounded"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "metis", "archived": True})) + with pytest.raises(project.ProjectPolicyError, match="archived"): + project._read_board("metis") + board_file.unlink() + board_file.mkdir() + with pytest.raises(project.ProjectPolicyError, match="regular"): + project._read_board("metis") + + +@pytest.mark.parametrize( + "remote", + [ + "https://token@scm.bstein.dev/atlas/metis.git", + "https://evil.example/atlas/metis.git", + "ssh://git@scm.bstein.dev/atlas/metis.git", + ], +) +def test_registry_rejects_credentialed_or_non_atlas_origin(tmp_path, monkeypatch, remote): + registry(tmp_path, monkeypatch, remote=remote) + with pytest.raises(project.ProjectPolicyError, match="origin"): + project.resolve_project("metis") + + +def test_registry_rejects_missing_or_outside_workdir(tmp_path, monkeypatch): + checkout, board_file = registry(tmp_path, monkeypatch) + board_file.write_text(json.dumps({"slug": "metis"})) + with pytest.raises(project.ProjectPolicyError, match="default_workdir"): + project.resolve_project("metis") + outside = tmp_path / "outside" + checkout.rename(outside) + board_file.write_text( + json.dumps({"slug": "metis", "default_workdir": str(outside)}) + ) + with pytest.raises(project.ProjectPolicyError, match="outside"): + project.resolve_project("metis") + + +def test_registry_rejects_non_directory_checkout(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + shutil.rmtree(checkout) + checkout.write_text("not a checkout") + with pytest.raises(project.ProjectPolicyError, match="not a directory"): + project.resolve_project("metis") + + +def test_git_metadata_failure_and_invalid_task_identity_fail_closed(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="metadata"): + project._run_git(checkout, "remote", "get-url", "missing") + with pytest.raises(project.ProjectPolicyError, match="identity"): + project.resolve_assignment("metis", SimpleNamespace(id="../../bad")) + + +def test_workspace_migration_boundary_is_explicit(): + assert project.distributed_workspace_eligible(SimpleNamespace(workspace_path="")) + assert project.distributed_workspace_eligible(SimpleNamespace()) + assert not project.distributed_workspace_eligible( + SimpleNamespace(workspace_path="/opt/data/workspace/live") + ) diff --git a/testing/tests/test_hermes_execution_pool_protocol_v2.py b/testing/tests/test_hermes_execution_pool_protocol_v2.py new file mode 100644 index 00000000..aea60727 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_protocol_v2.py @@ -0,0 +1,241 @@ +"""Version-2 protocol reliability and exact-attempt fencing contracts.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"m" * 32 + + +def binding(**changes): + value = { + "board": "metis", + "task_id": "t_deadbeef", + "run_id": "42", + "worker_ordinal": 0, + "attempt": 1, + } + value.update(changes) + return value + + +def resign(envelope, key=MASTER): + unsigned = dict(envelope) + unsigned.pop("signature", None) + envelope["signature"] = hmac.new( + key, protocol.canonical_json(unsigned), hashlib.sha256 + ).hexdigest() + return envelope + + +def test_atomic_json_is_private_durable_and_rejects_symlink_parent(tmp_path): + target = tmp_path / "state/value.json" + protocol.atomic_json(target, {"safe": True}) + assert json.loads(target.read_text()) == {"safe": True} + assert target.stat().st_mode & 0o777 == 0o600 + + outside = tmp_path / "outside" + outside.mkdir() + (tmp_path / "linked").symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="directory"): + protocol.atomic_json(tmp_path / "linked/value.json", {"safe": False}) + + +def test_ordinal_derivation_and_selector_fail_closed(): + keys = [protocol.derive_ordinal_key(MASTER, ordinal) for ordinal in range(3)] + assert len(set(keys)) == 3 + for master, ordinal in ((b"short", 0), (MASTER, -1), (MASTER, 3)): + with pytest.raises(protocol.ProtocolError, match="derivation"): + protocol.derive_ordinal_key(master, ordinal) + for value in (None, [], {}, {"worker_ordinal": True}, {"worker_ordinal": 3}): + with pytest.raises(protocol.ProtocolError): + protocol.envelope_ordinal(value) + + +def test_key_reader_bounds_content_and_type(tmp_path): + for content in (b"x" * 31, b"x" * 4097): + path = tmp_path / f"key-{len(content)}" + path.write_bytes(content) + path.chmod(0o600) + with pytest.raises(protocol.ProtocolError, match="length"): + protocol.read_key(path) + directory = tmp_path / "directory" + directory.mkdir(mode=0o700) + with pytest.raises(protocol.ProtocolError, match="regular"): + protocol.read_key(directory) + + +def test_envelope_rejects_kind_fields_numeric_lifetime_and_digest(): + with pytest.raises(protocol.ProtocolError, match="unsupported"): + protocol.sign_envelope(MASTER, "admin", binding(), {}) + valid = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + + extra = {**valid, "extra": True} + with pytest.raises(protocol.ProtocolError, match="fields"): + protocol.verify_envelope(MASTER, extra) + wrong_version = resign({**valid, "version": 1}) + with pytest.raises(protocol.ProtocolError, match="version"): + protocol.verify_envelope(MASTER, wrong_version) + with pytest.raises(protocol.ProtocolError, match="unexpected"): + protocol.verify_envelope(MASTER, valid, expected_kind="result") + + numeric = resign({**valid, "attempt": "not-a-number"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric) + numeric_text = resign({**valid, "attempt": "1"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric_text) + typed_identifier = resign({**valid, "run_id": 42}) + with pytest.raises(protocol.ProtocolError, match="run_id"): + protocol.verify_envelope(MASTER, typed_identifier) + lifetime = resign({**valid, "expires_at": valid["issued_at"]}) + with pytest.raises(protocol.ProtocolError, match="lifetime"): + protocol.verify_envelope(MASTER, lifetime) + digest = resign({**valid, "payload_digest": "0" * 64}) + with pytest.raises(protocol.ProtocolError, match="digest"): + protocol.verify_envelope(MASTER, digest) + with pytest.raises(protocol.ProtocolError, match="oversized"): + protocol.verify_envelope(MASTER, {"value": "x" * protocol.MAX_WIRE_BYTES}) + + +def test_wire_parser_rejects_non_object_and_empty(): + for body, message in ((b"", "empty"), (b"[]", "object")): + with pytest.raises(protocol.ProtocolError, match=message): + protocol.parse_wire(body) + + +def test_bounded_http_server_sets_timeout_and_releases_slot(): + handled = threading.Event() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + handled.set() + self.send_response(204) + self.end_headers() + + def log_message(self, *_args): + return + + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), Handler, max_workers=1) + thread = threading.Thread(target=server.handle_request) + thread.start() + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/", timeout=3 + ) as response: + assert response.status == 204 + thread.join(timeout=3) + assert handled.is_set() + acquired = False + for _ in range(100): + acquired = server._slots.acquire(blocking=False) + if acquired: + break + threading.Event().wait(0.01) + assert acquired + server._slots.release() + server.server_close() + + +def test_bounded_server_releases_slot_when_thread_dispatch_raises(monkeypatch): + server = protocol.BoundedHTTPServer( + ("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1 + ) + + def explode(*_args): + raise RuntimeError("dispatch failed") + + monkeypatch.setattr(ThreadingHTTPServer, "process_request", explode) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request(object(), ("127.0.0.1", 1)) + assert server._slots.acquire(blocking=False) + server._slots.release() + server.server_close() + + +def test_store_fences_ordinal_attempt_state_and_expired_lease(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db", lease_seconds=60) + assert protocol.PoolStore._record(None) is None + store.add(binding(), {"context": "safe"}) + store.offer(0) + + foreign = protocol.sign_envelope( + MASTER, "heartbeat", binding(worker_ordinal=1), {"note": "foreign"} + ) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + store.heartbeat(foreign) + + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + expired = protocol.sign_envelope(MASTER, "heartbeat", binding(), {"note": "late"}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.heartbeat(expired) + terminal = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.accept_result(terminal) + + +def test_store_pending_result_invalid_state_and_exact_finalize(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + result = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + record, duplicate = store.accept_result(result) + assert duplicate is False and record["state"] == "result" + assert store.pending_results()[0]["result"] == {"structured": {}} + with pytest.raises(protocol.ProtocolError, match="terminal"): + store.finalize(binding(), "invalid") + store.finalize(binding(attempt=2), "stale") + assert store.pending_results()[0]["state"] == "result" + store.finalize(binding(), "finalized") + heartbeat = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="running"): + store.heartbeat(heartbeat) + + +def test_expired_attempt_is_reoffered_then_terminally_released(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + store.offer(0) + changed = store.expire_leases(now=10_000_000_000, max_attempts=2) + assert changed[0]["state"] == "assigned" and changed[0]["attempt"] == 2 + stale = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="attempt is stale"): + store.heartbeat(stale) + + offered = store.offer(0) + assert offered and offered["attempt"] == 2 + changed = store.expire_leases(now=10_000_000_001, max_attempts=2) + assert changed[0]["state"] == "lease_failed" + assert store.available_ordinals() == [0, 1, 2] + + +def test_store_garbage_collection_removes_old_terminal_and_deliveries(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + heartbeat = protocol.sign_envelope( + MASTER, "heartbeat", binding(), {}, delivery_id="old-delivery" + ) + store.heartbeat(heartbeat) + store.finalize(binding(), "stale") + with store._connect() as connection: + connection.execute("UPDATE assignments SET updated_at=1") + connection.execute("UPDATE deliveries SET received_at=1") + assert store.garbage_collect(3600) == 1 + assert store.active_assignments() == [] + with store._connect() as connection: + assert connection.execute("SELECT count(*) FROM deliveries").fetchone()[0] == 0 diff --git a/testing/tests/test_hermes_execution_pool_scm_tampering.py b/testing/tests/test_hermes_execution_pool_scm_tampering.py new file mode 100644 index 00000000..50bfe9e8 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_scm_tampering.py @@ -0,0 +1,200 @@ +"""SCM checkout ownership and private-baseline corruption probes.""" + +from __future__ import annotations + +import sys +import subprocess +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +sys.path[:0] = [ + str(ROOT / "services/hermes/scripts"), + str(ROOT / "services/hermes/scm-common/scripts"), +] + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402 + KEY, + assignment, + init_checkout, +) + + +def test_checkout_and_submit_reject_unmanaged_or_corrupt_private_state( + tmp_path, monkeypatch +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + (destination / "owner-file").write_text("preserve") + boundary = scm.Boundary(KEY) + with pytest.raises(protocol.ProtocolError, match="non-empty"): + boundary.checkout(assignment()) + + (destination / "owner-file").unlink() + + def clone_with_empty_failure(*arguments, **_kwargs): + if arguments[0] == "clone" and arguments[4] == "wt/t_deadbeef": + destination.mkdir(parents=True, exist_ok=True) + raise RuntimeError("missing") + if arguments[0] == "clone": + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", clone_with_empty_failure) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + assert boundary.checkout(assignment())["baseline_sha"] == "a" * 40 + + protocol.atomic_json(scm._state_path(assignment()), {"baseline_sha": "bad"}) + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.submit(assignment(), {}) + + +def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path): + checkout = tmp_path / "checkout" + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "origin", + "https://scm.bstein.dev/atlas/titan-iac.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "hermes-broker", + scm._broker_repo("titan-iac"), + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "checkout", "-qb", + "feature/hermes-safe-pool", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.email", "a@b.c"], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.name", "Test"], + check=True, + ) + (checkout / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True) + subprocess.run( + ["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True + ) + expected = subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True + ).strip() + subprocess.run( + ["git", "-C", str(checkout), "pack-refs", "--all"], check=True + ) + assert scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) == expected + (checkout / ".git/refs/heads/feature").mkdir(parents=True) + (checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to( + "/etc/passwd" + ) + with pytest.raises(protocol.ProtocolError, match="symlink"): + scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) + + +def test_scm_paths_and_identity_reject_symlink_and_git_tampering( + tmp_path, monkeypatch +): + workspace = tmp_path / "workspace" + workspace.mkdir() + workspace_link = tmp_path / "workspace-link" + workspace_link.symlink_to(workspace, target_is_directory=True) + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace_link) + with pytest.raises(protocol.ProtocolError, match="root"): + scm.workspace_path(assignment()) + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + (workspace / "runs").mkdir() + (workspace / "runs/metis").symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="parent"): + scm.workspace_path(assignment()) + + state = tmp_path / "state" + state.mkdir() + monkeypatch.setattr(scm, "SCM_ROOT", state) + expected_state = state / "metis-t_deadbeef-42.json" + expected_state.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="state"): + scm._state_path(assignment()) + + checkout = tmp_path / "checkout" + checkout.mkdir() + with pytest.raises(protocol.ProtocolError, match="metadata"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + init_checkout(checkout) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://evil.example/repo.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="origin"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://scm.bstein.dev/atlas/metis.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/soteria.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="broker remote"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="branch"): + scm._workspace_identity(checkout, "metis", "review/other") + monkeypatch.setattr( + scm, + "_run", + lambda *arguments, **_kwargs: { + "remote": ( + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git" + if arguments[-1] == "hermes-broker" + else "https://scm.bstein.dev/atlas/metis.git" + ), + "symbolic-ref": "wt/t_deadbeef", + "rev-parse": "invalid", + }[arguments[0]], + ) + with pytest.raises(protocol.ProtocolError, match="HEAD"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") diff --git a/testing/tests/test_hermes_execution_pool_worker_execute.py b/testing/tests/test_hermes_execution_pool_worker_execute.py new file mode 100644 index 00000000..05b5add4 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_execute.py @@ -0,0 +1,246 @@ +"""Worker execution, fallback, terminal handoff, and exception contracts.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +from cli_lane_config import ProcessResult, Route # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 +from testing.tests.test_hermes_execution_pool_worker_v2 import assignment # noqa: E402 + + +def route(provider="codex", effort="high"): + return Route( + provider=provider, + model=f"{provider}-model", + effort=effort, + profile="agent", + classifier="test", + reason="unit", + latency_ms=1, + fallback_chain=(provider,), + ) + + +def completed_result(**changes): + structured = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], + } + value = ProcessResult( + returncode=0, + output="done", + structured=structured, + capacity_failure=False, + ) + for name, replacement in changes.items(): + setattr(value, name, replacement) + return value + + +def prepare_execute(tmp_path, monkeypatch, *, item=None, results=None, git_status=""): + worker_root = tmp_path / "worker" + workspace = worker_root / "runs/metis/t_deadbeef/23" + workspace.mkdir(parents=True) + exact = item or assignment( + workspace=str(workspace), + payload={ + "context": "safe objective", + "assignee": "cli-auto", + "deadline_unix": 10_000_000_000, + "max_runtime_seconds": 3600, + }, + ) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "NODE", "titan-05") + monkeypatch.setattr(worker, "_bind_provider_sessions", lambda _a: None) + monkeypatch.setattr(worker.cli_lane_runner, "load_json", lambda _path: {}) + routes = [route("codex"), route("claude")] + route_calls = [] + + def select_route(*arguments, **keywords): + route_calls.append((arguments, keywords)) + return routes[min(len(route_calls) - 1, len(routes) - 1)] + + monkeypatch.setattr(worker.cli_lane_runner, "select_route", select_route) + monkeypatch.setattr(worker.cli_lane_runner, "fresh_unavailable_provider", lambda: "claude") + monkeypatch.setattr(worker.cli_lane_runner, "git_handoff", lambda *_a: "\nhandoff") + outcomes = list(results or [completed_result()]) + provider_calls = [] + + def run_provider(*arguments): + provider_calls.append(arguments) + return outcomes.pop(0) + + monkeypatch.setattr(worker.cli_lane_runner, "run_provider", run_provider) + client_calls = [] + + def client(operation, **values): + client_calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + monkeypatch.setattr(worker, "_git", lambda *_a: git_status) + refreshed = [] + monkeypatch.setattr(worker, "_refresh_assignment", lambda exact: refreshed.append(exact)) + return exact, workspace, client_calls, provider_calls, route_calls, refreshed + + +def test_execute_completed_clean_result_refreshes_and_finishes_exact_run( + tmp_path, monkeypatch +): + exact, _workspace, calls, providers, routes, refreshed = prepare_execute( + tmp_path, monkeypatch + ) + worker.execute(exact) + assert len(providers) == 1 and len(routes) == 1 + assert refreshed == [worker._binding(exact)] + operations = [name for name, _values in calls] + assert operations == ["heartbeat", "finish"] + finish = calls[-1][1]["payload"] + assert finish["structured"]["status"] == "completed" + assert finish["node"] == "titan-05" + state = json.loads(worker._state_path(exact).read_text()) + assert state["terminal_at"] > 0 and state["baseline_sha"] == "a" * 40 + + +def test_execute_capacity_fallback_changes_provider_and_preserves_handoff( + tmp_path, monkeypatch +): + first = completed_result(capacity_failure=True, output="capacity") + second = completed_result() + exact, _workspace, calls, providers, routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[first, second] + ) + worker.execute(exact) + assert len(providers) == 2 and len(routes) == 2 + assert providers[1][0].provider == "claude" + assert providers[1][1].endswith("handoff") + heartbeats = [value for operation, value in calls if operation == "heartbeat"] + assert any("fallback=codex->claude" in value["payload"]["note"] for value in heartbeats) + + +def test_execute_dirty_workspace_downgrades_completed_result(tmp_path, monkeypatch): + exact, _workspace, calls, _providers, _routes, refreshed = prepare_execute( + tmp_path, monkeypatch, git_status="?? untracked" + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + assert structured["status"] == "incomplete" + assert "uncommitted or untracked" in structured["blockers"][0] + assert refreshed == [] + + +def test_execute_fills_missing_lists_for_failed_provider_result(tmp_path, monkeypatch): + failed = ProcessResult( + returncode=1, + output="failed", + structured={"status": "blocked", "summary": "Provider failed."}, + capacity_failure=False, + ) + exact, _workspace, calls, _providers, _routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[failed] + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + assert structured[name] == [] + + +def test_execute_rejects_payload_lease_baseline_and_terminal_ack_failures( + tmp_path, monkeypatch +): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + with pytest.raises(protocol.ProtocolError, match="payload"): + worker.execute({**exact, "payload": []}) + + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"ack": {"accepted": False}}) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "baseline", monkeypatch) + exact["baseline_sha"] = "invalid" + with pytest.raises(RuntimeError, match="baseline"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "ack", monkeypatch) + calls = [] + + def client(operation, **_values): + calls.append(operation) + return {"ack": {"accepted": operation != "finish"}} + + monkeypatch.setattr(worker, "_client", client) + with pytest.raises(protocol.ProtocolError, match="terminal"): + worker.execute(exact) + + +def test_execute_heartbeat_transport_failure_loses_lease(tmp_path, monkeypatch): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(ValueError("offline")) + ) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + +def test_report_exception_is_transient_exact_and_transport_safe(monkeypatch): + calls = [] + + def client(operation, **values): + calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + assert worker.report_exception(assignment(), RuntimeError("boom")) + payload = calls[0][1]["payload"] + assert payload["capacity_failure"] is True + assert payload["structured"]["status"] == "blocked" + assert payload["structured"]["blockers"] == ["RuntimeError: boom"] + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(OSError("offline")) + ) + assert worker.report_exception(assignment(), OSError("boom")) is False + + +def test_main_idle_and_exception_paths_do_not_spin_silently(monkeypatch, capsys): + monkeypatch.setattr(worker, "readiness", lambda: None) + monkeypatch.setattr(worker, "garbage_collect", lambda: 0) + monkeypatch.setattr(worker, "_poll", lambda: None) + monkeypatch.setattr( + worker.time, "sleep", lambda _seconds: (_ for _ in ()).throw(SystemExit("stop")) + ) + with pytest.raises(SystemExit, match="stop"): + worker.main() + + exact = assignment() + monkeypatch.setattr(worker, "_poll", lambda: exact) + monkeypatch.setattr(worker, "execute", lambda _a: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(worker, "report_exception", lambda _a, _e: True) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "surfaced" in capsys.readouterr().out + + monkeypatch.setattr( + worker, "garbage_collect", lambda: (_ for _ in ()).throw(RuntimeError("gc")) + ) + monkeypatch.setattr(worker, "report_exception", lambda *_a: False) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "deferred" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py new file mode 100644 index 00000000..a1984e56 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -0,0 +1,292 @@ +"""Model worker path, evidence, retention, and readiness contracts.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +def assignment(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, "protocol_version": 2, + "workspace": "/workspace/runs/metis/t_deadbeef/23", + "baseline_sha": "a" * 40, + "payload": {"context": "safe objective", "assignee": "cli-auto"}, + } + value.update(changes) + return value + + +class Response: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.value + + +def test_post_client_and_poll_validate_every_response_boundary(monkeypatch): + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(json.dumps({"safe": True}).encode()), + ) + assert worker._post("http://mediator", {}) == {"safe": True} + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(b"x" * (64 * 1024 + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="wire"): + worker._post("http://mediator", {}) + monkeypatch.setattr( + worker.urllib.request, "urlopen", lambda *_a, **_k: Response(b"[]") + ) + with pytest.raises(protocol.ProtocolError, match="object"): + worker._post("http://mediator", {}) + + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"error": "denied"}) + with pytest.raises(protocol.ProtocolError, match="denied"): + worker._client("poll") + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"safe": True}) + assert worker._client("poll") == {"safe": True} + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": None}) + assert worker._poll() is None + for value in ([], assignment(worker_ordinal=1), assignment(protocol_version=1)): + monkeypatch.setattr(worker, "_client", lambda *_a, value=value, **_k: {"assignment": value}) + with pytest.raises(protocol.ProtocolError, match="foreign"): + worker._poll() + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": assignment()}) + assert worker._poll()["task_id"] == "t_deadbeef" + assert worker._binding(assignment()) == { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + + +def test_state_path_rejects_traversal_root_and_leaf_symlinks(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + path = worker._state_path(assignment()) + assert path == tmp_path / "session-state/metis/t_deadbeef/23.json" + with pytest.raises(protocol.ProtocolError, match="invalid"): + worker._state_path(assignment(task_id="../bad")) + + state_root = tmp_path / "session-state" + outside = tmp_path / "outside" + path.parent.rmdir() + path.parent.parent.rmdir() + state_root.rmdir() + outside.mkdir() + state_root.symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="root"): + worker._state_path(assignment()) + state_root.unlink() + path = worker._state_path(assignment()) + path.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="symlink"): + worker._state_path(assignment()) + + +def prepare_provider_roots(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "runtime/codex" + claude = tmp_path / "runtime/claude" + (worker_root / "provider-state").mkdir(parents=True) + data_root.mkdir() + codex.mkdir(parents=True) + claude.mkdir(parents=True) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + return worker_root, data_root, codex, claude + + +def test_provider_session_binding_replaces_only_owned_symlinks(tmp_path, monkeypatch): + worker_root, data_root, codex, claude = prepare_provider_roots(tmp_path, monkeypatch) + old = tmp_path / "old" + old.mkdir() + (data_root / "home").symlink_to(old, target_is_directory=True) + (codex / "sessions").symlink_to(old, target_is_directory=True) + worker._bind_provider_sessions(assignment()) + assert "metis/t_deadbeef/23" in str((data_root / "home").resolve()) + assert "metis/t_deadbeef/23" in str((codex / "sessions").resolve()) + settings = worker_root / "provider-state/metis/t_deadbeef/23/home/.claude/settings.json" + assert json.loads(settings.read_text()) == {} + worker._bind_provider_sessions(assignment()) + + with pytest.raises(protocol.ProtocolError, match="binding"): + worker._bind_provider_sessions(assignment(run_id="../bad")) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="unavailable"): + worker._bind_provider_sessions(assignment()) + + +def test_provider_session_binding_rejects_durable_and_runtime_tampering( + tmp_path, monkeypatch +): + worker_root, data_root, codex, _claude = prepare_provider_roots(tmp_path, monkeypatch) + run_parent = worker_root / "provider-state/metis" + run_parent.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + run_parent.unlink() + home = worker_root / "provider-state/metis/t_deadbeef/23/home" + home.parent.mkdir(parents=True) + home.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="HOME"): + worker._bind_provider_sessions(assignment()) + home.unlink() + (data_root / "home").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="task-bound"): + worker._bind_provider_sessions(assignment()) + (data_root / "home").unlink() + (codex / "sessions").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="not a symlink"): + worker._bind_provider_sessions(assignment()) + (codex / "sessions").unlink() + durable_sessions = ( + worker_root / "provider-state/metis/t_deadbeef/23/codex/sessions" + ) + durable_sessions.rmdir() + durable_sessions.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + + +def test_prompt_activity_git_and_result_bounding(tmp_path, monkeypatch): + text = worker._prompt("objective", tmp_path, worker._binding(assignment())) + assert "no Kubernetes identity" in text and "objective" in text + missing = tmp_path / "missing.log" + assert worker._read_activity(missing, 7) == ("", 7) + log = tmp_path / "worker.log" + log.write_text("abc") + assert worker._read_activity(log, 0) == ("abc", 3) + assert worker._read_activity(log, 99) == ("abc", 3) + fifo = tmp_path / "fifo" + os.mkfifo(fifo) + with pytest.raises(protocol.ProtocolError, match="regular"): + worker._read_activity(fifo, 0) + + repo = tmp_path / "repo" + subprocess.run(["git", "init", "-q", str(repo)], check=True) + assert worker._git(repo, "status", "--porcelain") == "" + with pytest.raises(RuntimeError, match="ambiguous"): + worker._git(repo, "rev-parse", "missing") + + value = { + "status": "completed", "summary": "s" * 20_000, + "changed_files": ["x" * 3000] * 100, + "tests_run": "not-list", "artifacts": [], "findings": [], "blockers": [], + } + bounded = worker._bounded_result(value) + assert len(protocol.canonical_json(bounded)) <= 32 * 1024 + assert bounded["tests_run"] == [] and len(bounded["summary"]) <= 8000 + + +def test_refresh_assignment_requires_exact_binding(monkeypatch): + exact = worker._binding(assignment()) + monkeypatch.setattr(worker, "_poll", lambda: assignment()) + assert worker._refresh_assignment(exact)["run_id"] == "23" + for value in (None, assignment(attempt=2)): + monkeypatch.setattr(worker, "_poll", lambda value=value: value) + with pytest.raises(protocol.ProtocolError, match="changed"): + worker._refresh_assignment(exact) + + +def write_gc_state(root, task_id, workspace, terminal_at): + path = root / f"session-state/metis/{task_id}/23.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"terminal_at": terminal_at, "workspace": str(workspace)})) + return path + + +def init_clean_repo(path): + subprocess.run(["git", "init", "-q", str(path)], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.email", "a@b.c"], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.name", "Test"], check=True) + (path / "tracked").write_text("safe") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + + +def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600) + run_root = tmp_path / "runs/metis" + clean = run_root / "clean/23" + dirty = run_root / "dirty/23" + clean.mkdir(parents=True) + dirty.mkdir(parents=True) + init_clean_repo(clean) + init_clean_repo(dirty) + (dirty / "untracked").write_text("dirty") + now = 10_000 + write_gc_state(tmp_path, "clean", clean, 1) + dirty_state = write_gc_state(tmp_path, "dirty", dirty, 1) + young = write_gc_state(tmp_path, "young", clean, now) + outside = write_gc_state(tmp_path, "outside", tmp_path / "missing", 1) + assert worker.garbage_collect(now=now) == 1 + assert not clean.exists() and dirty.exists() + assert dirty_state.exists() and young.exists() and outside.exists() + + link = run_root / "link/23" + link.parent.mkdir() + link.symlink_to(dirty, target_is_directory=True) + link_state = write_gc_state(tmp_path, "link", link, 1) + assert worker.garbage_collect(now=now) == 0 + assert link_state.exists() + + +def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "codex" + claude = tmp_path / "claude" + for path in (worker_root, worker_root / "provider-state", data_root, codex, claude): + path.mkdir(parents=True, exist_ok=True) + (codex / "auth.json").write_text("{}") + (claude / ".credentials.json").write_text("{}") + schema = tmp_path / "schema/result.json" + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + polled = [] + monkeypatch.setattr(worker, "_poll", lambda: polled.append(True)) + worker.readiness() + assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA + + monkeypatch.setattr(worker, "ORDINAL", 3) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + worker.readiness() + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="path"): + worker.readiness() + monkeypatch.setattr(worker, "ROOT", worker_root) + (codex / "auth.json").unlink() + with pytest.raises(protocol.ProtocolError, match="credential"): + worker.readiness() diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index ee482cfe..b9f44bc4 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -142,7 +142,7 @@ def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): ) ) boundary = common["configMapGenerator"][0] - assert boundary["name"] == "hermes-scm-boundary" + assert boundary["name"] == "hermes-scm-boundary-v2" assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py index 3370fe4f..a8762e54 100644 --- a/testing/tests/test_hermes_node_account_privilege_audit.py +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil import subprocess from pathlib import Path @@ -115,8 +116,11 @@ def test_visudo_valid_numeric_and_alias_grants_fail_closed( monkeypatch.setattr(module, "ACCOUNT_UID", 1200) policy = module.HOST_ETC / "sudoers" policy.write_text(value, encoding="utf-8") + visudo = shutil.which("visudo") + if visudo is None: + pytest.skip("visudo is not installed in this test environment") validation = subprocess.run( - ["/usr/bin/visudo", "-c", "-f", str(policy)], + [visudo, "-c", "-f", str(policy)], check=False, capture_output=True, text=True, diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 1ffb6ac7..21054868 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import hashlib +import hmac import json import sys import urllib.request @@ -172,6 +174,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp ) values = { "agent-api-key": "agent-key", + "execution-pool-key": "e" * 64, "chat-relay-key": "relay-key", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", @@ -204,6 +207,89 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert auth == {"version": 1, "providers": {}, "credential_pool": {}} +def test_execution_worker_and_mediator_separate_credentials_and_hmac( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + vault = tmp_path / "vault" + runtime = tmp_path / "runtime" + worker = tmp_path / "worker" + provider_access = tmp_path / "provider-access" + pool_access = tmp_path / "pool-access" + vault.mkdir() + (vault / "claude-credentials-1").write_text( + json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) + ) + (vault / "codex-auth-1").write_text( + json.dumps({"tokens": {"refresh_token": "codex-refresh"}}) + ) + monkeypatch.setattr(stage, "VAULT_ROOT", vault) + monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) + monkeypatch.setattr(stage, "WORKER_ROOT", worker) + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider_access) + monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", pool_access) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "1") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) + + stage.stage_execution_worker() + + assert not pool_access.exists() + assert not (runtime / "execution-pool-key").exists() + assert not (runtime / "codex/sessions").exists() + assert not (runtime / "claude/projects").exists() + assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + refreshed = {"tokens": {"refresh_token": "provider-rotated"}} + (provider_access / "codex/auth.json").write_text(json.dumps(refreshed)) + (provider_access / "codex/auth.json").chmod(0o600) + stage.stage_execution_worker() + assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed + + master = "e" * 64 + (vault / "execution-pool-key").write_text(master) + stage.stage_execution_mediator() + expected = hmac.new( + master.encode(), b"hermes-execution-pool-v2:worker:1", hashlib.sha256 + ).hexdigest() + assert (pool_access / "execution-pool-key").read_text().strip() == expected + + +def test_execution_worker_fails_closed_without_channel_credential( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + vault = tmp_path / "vault" + vault.mkdir() + (vault / "claude-credentials-0").write_text( + json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) + ) + monkeypatch.setattr(stage, "VAULT_ROOT", vault) + monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime") + monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", tmp_path / "provider-access") + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) + + with pytest.raises(FileNotFoundError): + stage.stage_execution_worker() + + +def test_execution_worker_rejects_durable_credential_symlink( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + provider = tmp_path / "provider" + provider.mkdir() + (provider / "claude").symlink_to(tmp_path, target_is_directory=True) + monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + with pytest.raises(RuntimeError, match="symlink"): + stage.stage_execution_worker() + + def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): stage = _load("stage_runtime_access") vault = tmp_path / "vault" @@ -217,7 +303,7 @@ def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): try: stage._validated_json("credential", destination, ("token",)) - except json.JSONDecodeError: + except RuntimeError: pass else: raise AssertionError("invalid credential JSON should fail staging") diff --git a/testing/tests/test_hermes_runtime_stage_coverage.py b/testing/tests/test_hermes_runtime_stage_coverage.py index 004853c2..3ca8e496 100644 --- a/testing/tests/test_hermes_runtime_stage_coverage.py +++ b/testing/tests/test_hermes_runtime_stage_coverage.py @@ -39,6 +39,21 @@ def test_private_directory_and_secret_copy_validate_mode_and_content( assert (runtime / "value").read_text(encoding="utf-8") == "staged\n" +def test_private_directory_and_vault_projection_reject_invalid_entries( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + checks = iter((False, True)) + with monkeypatch.context() as invalid_directory: + invalid_directory.setattr(Path, "is_symlink", lambda _path: next(checks)) + with pytest.raises(RuntimeError, match="is invalid"): + module._owned_directory(runtime) + + (vault / "invalid").write_bytes(b"") + with pytest.raises(RuntimeError, match="projection invalid is invalid"): + module._read_secret("invalid") + + @pytest.mark.parametrize( ("document", "required", "match"), [ @@ -107,6 +122,77 @@ def test_node_ssh_config_rejects_nul_and_removes_partial_file(tmp_path, monkeypa assert not (runtime / "node-ssh-config").exists() +def test_durable_credentials_reject_links_and_unsafe_existing_files( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "credential").write_text( + json.dumps({"tokens": {"refresh_token": "synthetic"}}), + encoding="utf-8", + ) + destination = runtime / "credential.json" + destination.symlink_to(vault / "credential") + with pytest.raises(RuntimeError, match="must not be a symlink"): + module._bootstrap_json("credential", destination, ("tokens", "refresh_token")) + + destination.unlink() + destination.write_text( + json.dumps({"tokens": {"refresh_token": "durable"}}), + encoding="utf-8", + ) + destination.chmod(0o644) + with pytest.raises(RuntimeError, match="is unsafe"): + module._bootstrap_json("credential", destination, ("tokens", "refresh_token")) + + +def test_durable_link_creates_repairs_preserves_and_rejects_targets( + tmp_path, monkeypatch +): + module, _vault, _runtime, _home = _stage(tmp_path, monkeypatch) + monkeypatch.setattr(module.os, "chown", lambda *_args: None) + runtime = tmp_path / "worker" + runtime.mkdir() + durable = tmp_path / "provider" + + module._durable_link(runtime, durable, "state") + target = runtime / "state" + assert target.is_symlink() and target.resolve() == durable.resolve() + module._durable_link(runtime, durable, "state") + assert target.is_symlink() + + target.unlink() + wrong = tmp_path / "wrong" + wrong.mkdir() + target.symlink_to(wrong) + module._durable_link(runtime, durable, "state") + assert target.resolve() == durable.resolve() + + target.unlink() + target.write_text("collision", encoding="utf-8") + with pytest.raises(RuntimeError, match="refusing to replace"): + module._durable_link(runtime, durable, "state") + + +@pytest.mark.parametrize( + ("function", "match"), + [ + ("stage_execution_worker", "worker ordinal is invalid"), + ("stage_execution_mediator", "mediator ordinal is invalid"), + ], +) +def test_execution_staging_rejects_invalid_ordinal( + tmp_path, monkeypatch, function, match +): + module, _vault, _runtime, _home = _stage(tmp_path, monkeypatch) + monkeypatch.setattr(module, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(module, "PROVIDER_ACCESS_ROOT", tmp_path / "provider") + monkeypatch.setattr(module, "POOL_ACCESS_ROOT", tmp_path / "pool") + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "9") + with pytest.raises(RuntimeError, match=match): + getattr(module, function)() + + @pytest.mark.parametrize( ("mode", "secret_name"), [("chat", "chat-relay-key"), ("triage", "triage-api-key")], @@ -124,13 +210,21 @@ def test_isolated_tenant_staging_exposes_only_required_secret( ) -@pytest.mark.parametrize("mode", ["agent", "chat", "triage"]) +@pytest.mark.parametrize( + "mode", ["agent", "chat", "triage", "execution-worker", "execution-mediator"] +) def test_main_dispatches_each_mode(mode, monkeypatch, capsys): module = _load("stage_runtime_access") calls = [] monkeypatch.setattr(module, "stage_agent", lambda: calls.append("agent")) monkeypatch.setattr(module, "stage_chat", lambda: calls.append("chat")) monkeypatch.setattr(module, "stage_triage", lambda: calls.append("triage")) + monkeypatch.setattr( + module, "stage_execution_worker", lambda: calls.append("execution-worker") + ) + monkeypatch.setattr( + module, "stage_execution_mediator", lambda: calls.append("execution-mediator") + ) monkeypatch.setattr(sys, "argv", ["stage-runtime-access", mode]) assert module.main() == 0 assert calls == [mode] diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py index 2a0fad44..8f1b2ca9 100644 --- a/testing/tests/test_hermes_scm_broker.py +++ b/testing/tests/test_hermes_scm_broker.py @@ -48,7 +48,7 @@ def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): api.authorize_request("GET", path, None) -def test_receive_pack_allows_only_new_namespaced_feature_branch(): +def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces(): broker = _load("scm_broker") zero = b"0" * 40 commit = b"1" * 40 @@ -57,9 +57,18 @@ def test_receive_pack_allows_only_new_namespaced_feature_branch(): _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), "runtime-sentinel", ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/wt/t_deadbeef"), + "runtime-sentinel", + ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/review/t_deadbeef"), + "runtime-sentinel", + ) for old, new, ref in ( (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), (commit, zero, b"refs/heads/hermes/focused-fix"), + (commit, commit, b"refs/heads/hermes/focused-fix"), (zero, commit, b"refs/heads/main"), (zero, commit, b"refs/heads/master"), (zero, commit, b"refs/tags/release"), From 5d963c9354bbcd91d54d1ea45bcebd603000fe2b Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:31:15 +0000 Subject: [PATCH 21/46] hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 --- .../applications/hermes/kustomization.yaml | 10 +- services/hermes/execution-mediator.yaml | 81 +++- .../hermes/execution-worker-statefulset.yaml | 39 +- services/hermes/kustomization.yaml | 2 + .../hermes/scripts/execution_pool_client.py | 34 +- .../scripts/execution_pool_coordinator.py | 195 +-------- .../scripts/execution_pool_maintenance.py | 371 +++++++++++++++++ .../hermes/scripts/execution_pool_protocol.py | 266 +----------- services/hermes/scripts/execution_pool_scm.py | 70 +++- .../hermes/scripts/execution_pool_server.py | 47 ++- .../hermes/scripts/execution_pool_store.py | 321 ++++++++++++++ testing/quality_contract.json | 8 + testing/tests/test_hermes_execution_pool.py | 13 +- .../test_hermes_execution_pool_dispatch_v2.py | 13 +- .../test_hermes_execution_pool_isolation.py | 391 ++++++++++++++++++ .../test_hermes_execution_pool_manifests.py | 83 ++++ .../test_hermes_execution_pool_placement.py | 197 +++++++++ .../test_hermes_execution_pool_recovery.py | 175 ++++++++ .../test_hermes_execution_pool_submission.py | 253 ++++++++++++ .../test_hermes_execution_pool_support.py | 60 +++ 20 files changed, 2133 insertions(+), 496 deletions(-) create mode 100644 services/hermes/scripts/execution_pool_maintenance.py create mode 100644 services/hermes/scripts/execution_pool_store.py create mode 100644 testing/tests/test_hermes_execution_pool_isolation.py create mode 100644 testing/tests/test_hermes_execution_pool_manifests.py create mode 100644 testing/tests/test_hermes_execution_pool_placement.py create mode 100644 testing/tests/test_hermes_execution_pool_recovery.py create mode 100644 testing/tests/test_hermes_execution_pool_submission.py create mode 100644 testing/tests/test_hermes_execution_pool_support.py diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 084e6074..09246661 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -30,10 +30,12 @@ spec: kind: Deployment name: hermes-agent namespace: hermes - - apiVersion: apps/v1 - kind: StatefulSet - name: hermes-execution-worker - namespace: hermes + # hermes-execution-worker and its mediators are deliberately absent. They run + # at scavenger priority on a best-effort pool whose first start installs the + # provider CLIs, so gating this Kustomization's 10m health window on them + # would stall hermes-chat and hermes-observer-bindings, which dependsOn + # hermes. The pool reports its own health through the worker readiness probe, + # the per-ordinal mediator /ready endpoint, and the coordinator on :9007. - apiVersion: apps/v1 kind: DaemonSet name: hermes-node-ssh-access diff --git a/services/hermes/execution-mediator.yaml b/services/hermes/execution-mediator.yaml index d41ee9f9..4970aa4f 100644 --- a/services/hermes/execution-mediator.yaml +++ b/services/hermes/execution-mediator.yaml @@ -83,17 +83,34 @@ spec: serviceAccountName: hermes-execution-worker automountServiceAccountToken: false enableServiceLinks: false + # Priority parity with the worker: a pressured node evicts the pair + # symmetrically instead of moving only the lower-priority half, and the + # pool never preempts anything else in the cluster. + priorityClassName: scavenger securityContext: fsGroup: 10000 fsGroupChangePolicy: OnRootMismatch seccompProfile: {type: RuntimeDefault} affinity: - podAffinity: + # Colocation with the worker is a throughput preference, not a + # requirement. The shared workspace claim is ReadWriteMany, so this + # mediator still schedules and serves when its worker is Pending or + # placed elsewhere -- which is what makes a drain self-healing. + nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - statefulset.kubernetes.io/pod-name: hermes-execution-worker-0 - topologyKey: kubernetes.io/hostname + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/arch, operator: In, values: [arm64]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-0 + topologyKey: kubernetes.io/hostname initContainers: - name: stage-mediator-access image: registry.bstein.dev/bstein/hermes-agent @@ -209,17 +226,34 @@ spec: serviceAccountName: hermes-execution-worker automountServiceAccountToken: false enableServiceLinks: false + # Priority parity with the worker: a pressured node evicts the pair + # symmetrically instead of moving only the lower-priority half, and the + # pool never preempts anything else in the cluster. + priorityClassName: scavenger securityContext: fsGroup: 10000 fsGroupChangePolicy: OnRootMismatch seccompProfile: {type: RuntimeDefault} affinity: - podAffinity: + # Colocation with the worker is a throughput preference, not a + # requirement. The shared workspace claim is ReadWriteMany, so this + # mediator still schedules and serves when its worker is Pending or + # placed elsewhere -- which is what makes a drain self-healing. + nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - statefulset.kubernetes.io/pod-name: hermes-execution-worker-1 - topologyKey: kubernetes.io/hostname + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/arch, operator: In, values: [arm64]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-1 + topologyKey: kubernetes.io/hostname initContainers: - name: stage-mediator-access image: registry.bstein.dev/bstein/hermes-agent @@ -335,17 +369,34 @@ spec: serviceAccountName: hermes-execution-worker automountServiceAccountToken: false enableServiceLinks: false + # Priority parity with the worker: a pressured node evicts the pair + # symmetrically instead of moving only the lower-priority half, and the + # pool never preempts anything else in the cluster. + priorityClassName: scavenger securityContext: fsGroup: 10000 fsGroupChangePolicy: OnRootMismatch seccompProfile: {type: RuntimeDefault} affinity: - podAffinity: + # Colocation with the worker is a throughput preference, not a + # requirement. The shared workspace claim is ReadWriteMany, so this + # mediator still schedules and serves when its worker is Pending or + # placed elsewhere -- which is what makes a drain self-healing. + nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - statefulset.kubernetes.io/pod-name: hermes-execution-worker-2 - topologyKey: kubernetes.io/hostname + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/arch, operator: In, values: [arm64]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-2 + topologyKey: kubernetes.io/hostname initContainers: - name: stage-mediator-access image: registry.bstein.dev/bstein/hermes-agent diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml index 6794f4da..33f13f23 100644 --- a/services/hermes/execution-worker-statefulset.yaml +++ b/services/hermes/execution-worker-statefulset.yaml @@ -144,13 +144,24 @@ spec: imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: + # The tools volume is durable, so this install runs once per pinned + # version rather than on every Pod start. The version marker is only + # trusted when both binaries are actually present and executable, so + # a partial or pruned cache reinstalls instead of failing closed + # forever. The install itself is time-bounded so a stalled registry + # surfaces as a fast init failure with backoff instead of an + # open-ended wait that a Flux health timeout would have to absorb. - | tools=/worker-data/tools + marker="${tools}/.cli-versions-0.147.0-2.1.226" mkdir -p "${tools}/bin" - if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then - npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ + if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \ + || [ ! -x "${tools}/bin/claude" ]; then + rm -f "${marker}" + timeout 900 npm install --global --omit=dev --no-audit --no-fund \ + --fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \ @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226 - touch "${tools}/.cli-versions-0.147.0-2.1.226" + touch "${marker}" fi test -x "${tools}/bin/codex" test -x "${tools}/bin/claude" @@ -218,8 +229,6 @@ spec: volumes: - name: worker-data emptyDir: {sizeLimit: 128Mi} - - name: tools - emptyDir: {sizeLimit: 1Gi} - name: coordinator configMap: name: hermes-execution-pool @@ -242,16 +251,34 @@ spec: items: - {path: namespace, fieldRef: {fieldPath: metadata.namespace}} volumeClaimTemplates: + # The workspace is shared with this ordinal's mediator, which performs every + # SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach + # deadlock that a drain or preemption used to create between the two Pods, + # so each can be evicted and rescheduled independently. Longhorn already + # serves the hermes-chat tenant workspaces this way on the same class. - metadata: name: workspace labels: app: hermes-execution-worker + spec: + accessModes: [ReadWriteMany] + storageClassName: astreae + resources: + requests: + storage: 30Gi + # Provider CLIs are installed once per pinned version onto a durable volume + # instead of being re-downloaded into an emptyDir on every Pod start. + - metadata: + name: tools + labels: + app: hermes-execution-worker spec: accessModes: [ReadWriteOnce] storageClassName: astreae resources: requests: - storage: 30Gi + storage: 2Gi + # OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce. - metadata: name: provider-access labels: diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 6becbaf6..e0dfc35c 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -83,8 +83,10 @@ configMapGenerator: - cli_lane_routing.py=scripts/cli_lane_routing.py - cli_lane_runner.py=scripts/cli_lane_runner.py - execution_pool_protocol.py=scripts/execution_pool_protocol.py + - execution_pool_store.py=scripts/execution_pool_store.py - execution_pool_project.py=scripts/execution_pool_project.py - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py + - execution_pool_maintenance.py=scripts/execution_pool_maintenance.py - execution_pool_server.py=scripts/execution_pool_server.py - execution_pool_client.py=scripts/execution_pool_client.py - execution_pool_worker.py=scripts/execution_pool_worker.py diff --git a/services/hermes/scripts/execution_pool_client.py b/services/hermes/scripts/execution_pool_client.py index a1dc6622..2d376c48 100644 --- a/services/hermes/scripts/execution_pool_client.py +++ b/services/hermes/scripts/execution_pool_client.py @@ -137,16 +137,42 @@ class ClientBoundary: raise ProtocolError("coordinator acknowledgement binding changed") return {"ack": response["payload"]} + def _submit( + self, assignment: dict[str, Any], request: dict[str, Any], + structured: dict[str, Any], + ) -> None: + """Publish the run's work, downgrading the result rather than losing it. + + A refused push or draft used to unwind the whole worker, so the run's + actual output was discarded and the task was blocked with only the SCM + error. The commits live on the durable workspace volume, so the result is + still reported here -- as blocked, carrying the reason -- which keeps the + prior work recoverable and reviewable by a human. + """ + try: + submission = self.scm.submit(assignment, request) + except (OSError, ProtocolError, RuntimeError, ValueError) as error: + structured["status"] = "blocked" + reason = ( + "Distributed SCM submission failed; the commits remain on this " + f"ordinal's workspace: {type(error).__name__}: {error}" + ) + if reason not in structured["blockers"]: + structured["blockers"].append(reason) + return + pull = str(submission.get("pull_request") or "") + branch = str(submission.get("branch") or "") + for artifact in (pull, f"branch:{branch}" if branch else ""): + if artifact and artifact not in structured["artifacts"]: + structured["artifacts"].append(artifact) + def finish(self, request: dict[str, Any]) -> dict[str, Any]: payload = _validate_result(request.get("payload")) with self.lock: assignment, binding = self._current_for(request.get("binding")) structured = payload["structured"] if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0: - submission = self.scm.submit(assignment, request) - pull = str(submission.get("pull_request") or "") - if pull and pull not in structured["artifacts"]: - structured["artifacts"].append(pull) + self._submit(assignment, request, structured) response = self._post( "/v1/result", sign_envelope(self.key, "result", binding, payload) ) diff --git a/services/hermes/scripts/execution_pool_coordinator.py b/services/hermes/scripts/execution_pool_coordinator.py index 6fc0f5a4..1f4b0e85 100644 --- a/services/hermes/scripts/execution_pool_coordinator.py +++ b/services/hermes/scripts/execution_pool_coordinator.py @@ -6,8 +6,6 @@ from __future__ import annotations import json import os import re -import sqlite3 -import sys import threading import time from http.server import BaseHTTPRequestHandler @@ -15,22 +13,17 @@ from pathlib import Path from typing import Any import cli_lane_goal -import cli_lane_dispatch from cli_lane_config import canonical_run_id -from execution_pool_project import ( - ProjectPolicyError, - distributed_workspace_eligible, - resolve_assignment, -) +from execution_pool_project import resolve_assignment from execution_pool_protocol import ( MAX_ACTIVITY_BYTES, - PoolStore, ProtocolError, derive_ordinal_key, envelope_ordinal, sign_envelope, verify_envelope, ) +from execution_pool_store import PoolStore REDACTIONS = ( re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"), @@ -274,186 +267,36 @@ class Coordinator: finally: connection.close() + @staticmethod + def assignment_payload( + kanban_db: Any, connection: Any, task: Any, board: str + ) -> dict[str, Any]: + """Expose payload construction to the maintenance passes.""" + return assignment_payload(kanban_db, connection, task, board) + def recover_results(self) -> None: - for record in self.store.pending_results(): - try: - self.finalize(record) - except (OSError, sqlite3.Error) as error: - print(f"result recovery deferred: {error}", file=sys.stderr, flush=True) + """Re-apply every durable result that Kanban has not confirmed yet.""" + from execution_pool_maintenance import recover_results + + recover_results(self) def expire_leases(self) -> None: """Fence dead attempts; after bounded retries, surface and release the run.""" - from hermes_cli import kanban_db + from execution_pool_maintenance import expire_leases - for record in self.store.expire_leases(): - if record.get("state") != "lease_failed": - continue - binding = self._binding(record) - run_id = canonical_run_id(binding["run_id"]) - if run_id is None: - self.store.finalize(binding, "stale") - continue - with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): - connection = kanban_db.connect(board=binding["board"]) - try: - changed = kanban_db.block_task( - connection, - binding["task_id"], - reason=( - "Distributed worker lease expired after " - f"{binding['attempt']} fenced attempts" - ), - kind="transient", - expected_run_id=run_id, - ) - self.store.finalize( - binding, "finalized" if changed else "stale" - ) - finally: - connection.close() + expire_leases(self) def reconcile(self) -> None: """Recover the narrow claim/assignment crash gaps without double execution.""" - from hermes_cli import kanban_db + from execution_pool_maintenance import reconcile - active = self.store.active_assignments() - active_runs = { - (record["board"], record["task_id"], record["run_id"]) - for record in active - } - for record in active: - try: - with kanban_db.scoped_current_board(record["board"]): - connection = kanban_db.connect(board=record["board"]) - try: - task = kanban_db.get_task(connection, record["task_id"]) - current_run_id = ( - canonical_run_id(_task_value(task, "current_run_id", None)) - if task - else None - ) - current = str(current_run_id or "") - status = str(_task_value(task, "status", "") or "") if task else "" - finally: - connection.close() - if current != record["run_id"] or status != "running": - self.store.finalize(record, "stale") - except (OSError, sqlite3.Error): - continue - ordinals = self.store.available_ordinals() - if not ordinals: - return - for raw_board in kanban_db.list_boards(include_archived=False): - board = cli_lane_dispatch._board_slug(raw_board) - if not board: - continue - with kanban_db.scoped_current_board(board): - connection = kanban_db.connect(board=board) - try: - tasks = kanban_db.list_tasks(connection) - for task in tasks: - task_id = str(_task_value(task, "id", "") or "") - raw_run_id = canonical_run_id( - _task_value(task, "current_run_id", None) - ) - run_id = str(raw_run_id or "") - assignee = str(_task_value(task, "assignee", "") or "") - if ( - not ordinals - or not task_id - or not run_id - or str(_task_value(task, "status", "")) != "running" - or not assignee.startswith("cli-") - or not distributed_workspace_eligible(task) - or (board, task_id, run_id) in active_runs - ): - continue - try: - payload = assignment_payload(kanban_db, connection, task, board) - except Exception as error: - kanban_db.block_task( - connection, task_id, - reason=( - "Distributed assignment recovery failed: " - f"{type(error).__name__}: {error}" - ), - kind="capability", expected_run_id=raw_run_id, - ) - continue - ordinal = ordinals.pop(0) - self.store.add( - { - "board": board, "task_id": task_id, - "run_id": run_id, "worker_ordinal": ordinal, - "attempt": 1, - }, - payload, - ) - finally: - connection.close() + reconcile(self) def dispatch(self) -> None: """Claim up to the free ordinal count and materialize assignments.""" - from hermes_cli import kanban_db + from execution_pool_maintenance import dispatch - ordinals = self.store.available_ordinals() - if not ordinals: - return - claimed = cli_lane_dispatch.claim_ready( - set(), len(ordinals), - lambda _board, task: distributed_workspace_eligible(task), - ) - for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False): - run_id = "" - database_run_id: int | None = None - try: - with kanban_db.scoped_current_board(board): - connection = kanban_db.connect(board=board) - try: - task = kanban_db.get_task(connection, task_id) - if task is None: - continue - if not distributed_workspace_eligible(task): - raw_run = canonical_run_id( - _task_value(task, "current_run_id", None) - ) - if raw_run is not None: - kanban_db.block_task( - connection, task_id, - reason=( - "Distributed claim fenced because an existing " - "workspace is owned by the local lane" - ), - kind="capability", - expected_run_id=raw_run, - ) - continue - raw_run = canonical_run_id( - _task_value(task, "current_run_id", None) - ) - if raw_run is None: - raise ProjectPolicyError("claimed task has no canonical run ID") - database_run_id = raw_run - run_id = str(raw_run) - payload = assignment_payload(kanban_db, connection, task, board) - finally: - connection.close() - binding = { - "board": board, "task_id": task_id, "run_id": run_id, - "worker_ordinal": ordinal, "attempt": 1, - } - self.store.add(binding, payload) - except Exception as error: - reason = f"Distributed assignment preparation failed: {type(error).__name__}: {error}" - with kanban_db.scoped_current_board(board): - connection = kanban_db.connect(board=board) - try: - kanban_db.block_task( - connection, task_id, reason=reason, kind="capability", - expected_run_id=database_run_id, - ) - finally: - connection.close() + dispatch(self) def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]: diff --git a/services/hermes/scripts/execution_pool_maintenance.py b/services/hermes/scripts/execution_pool_maintenance.py new file mode 100644 index 00000000..c3d3d1d5 --- /dev/null +++ b/services/hermes/scripts/execution_pool_maintenance.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Recoverable maintenance passes for the Hermes execution-pool coordinator. + +Every pass here runs against a shared Kanban SQLite database that other writers +(the agent UI, the local CLI lane, crons) hold concurrently, so *any* board call +can fail transiently. The rules this module enforces: + +* No pass ever leaves a claimed run without a durable row that some later pass + will revisit. A Kanban write that fails leaves the row in its retryable state. +* No single poisoned row, task, or board aborts the work queued behind it. +* A durable row only reaches a terminal state on authoritative evidence about + its Kanban run, because terminal rows are the only ones ever collected. +* A coordinator-side fault is never converted into a Kanban mutation. Only a + genuine capability failure of the run itself parks that exact run. +""" + +from __future__ import annotations + +import sqlite3 +import sys +from typing import Any + +import cli_lane_dispatch +from cli_lane_config import canonical_run_id +from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible +from execution_pool_protocol import ProtocolError + + +RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error) +DEFERRALS: dict[str, str] = {} +MAX_DEFERRALS = 64 + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def _defer(context: str, error: BaseException) -> None: + """Report one recoverable pool-side failure once per distinct detail. + + The maintenance loop re-runs every few seconds, so an unchanged failure is + reported once rather than on every pass. The table is bounded because its + keys include run identities. + """ + detail = f"{type(error).__name__}: {error}" + if DEFERRALS.get(context) == detail: + return + if len(DEFERRALS) >= MAX_DEFERRALS: + DEFERRALS.clear() + DEFERRALS[context] = detail + print(f"pool work deferred for {context}: {detail}", file=sys.stderr, flush=True) + + +def _settled(context: str) -> None: + DEFERRALS.pop(context, None) + + +def _context(record: dict[str, Any]) -> str: + return f"{record['board']}/{record['task_id']}#{record['run_id']}" + + +def _block_exact( + kanban_db: Any, + board: str, + task_id: str, + run_id: int | None, + reason: str, + *, + kind: str = "capability", +) -> None: + """Park exactly one run, never letting the write itself escape.""" + try: + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + kanban_db.block_task( + connection, task_id, reason=reason, kind=kind, + expected_run_id=run_id, + ) + finally: + connection.close() + except Exception as error: # noqa: BLE001 - a failed park must not wedge the pass + _defer(f"{board}/{task_id} block", error) + + +def recover_results(pool: Any) -> None: + """Re-apply each durable result in isolation from its neighbours.""" + for record in pool.store.pending_results(): + try: + pool.finalize(record) + _settled(_context(record)) + except Exception as error: # noqa: BLE001 - one bad result must not block the rest + print( + f"result recovery deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + + +def _lease_failure_state( + kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int +) -> str: + """Classify one exhausted lease from authoritative Kanban evidence only.""" + task = kanban_db.get_task(connection, binding["task_id"]) + current = ( + canonical_run_id(_task_value(task, "current_run_id", None)) + if task is not None + else None + ) + if current != run_id: + return "stale" + if str(_task_value(task, "status", "") or "") != "running": + return "finalized" + if kanban_db.block_task( + connection, + binding["task_id"], + reason=( + "Distributed worker lease expired after " + f"{binding['attempt']} fenced attempts" + ), + kind="transient", + expected_run_id=run_id, + ): + return "finalized" + # Kanban refused the exact-run park yet still reports the run as running, so + # there is no terminal evidence. Keep the row for the next pass to retry. + raise ProtocolError("Kanban refused the exact-run lease park") + + +def record_lease_failure(pool: Any, record: dict[str, Any]) -> None: + """Record one exhausted lease against its exact Kanban run, idempotently. + + The row stays ``lease_failed`` unless this pass obtains authoritative + evidence: the run moved on or vanished (``stale``), Kanban already parked it + (``finalized``), or this call parked it (``finalized``). Anything else -- + including a refused write or an unreadable board -- leaves the row for the + next pass, so no run is ever silently dropped. + """ + from hermes_cli import kanban_db + + binding = pool._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + pool.store.finalize(binding, "stale") + return + with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + state = _lease_failure_state(kanban_db, connection, binding, run_id) + finally: + connection.close() + pool.store.finalize(binding, state) + _settled(_context(record)) + + +def recover_lease_failures(pool: Any) -> None: + """Retry every released-but-unrecorded lease failure, each in isolation.""" + for record in pool.store.failed_leases(): + try: + record_lease_failure(pool, record) + except Exception as error: # noqa: BLE001 - retried on the next pass + _defer(f"{_context(record)} lease park", error) + + +def expire_leases(pool: Any) -> None: + """Fence dead attempts, then drain every unconfirmed terminal record.""" + pool.store.expire_leases() + recover_lease_failures(pool) + + +def _release_moved_run(pool: Any, kanban_db: Any, record: dict[str, Any]) -> None: + """Release one durable assignment whose Kanban run is no longer ours.""" + with kanban_db.scoped_current_board(record["board"]): + connection = kanban_db.connect(board=record["board"]) + try: + task = kanban_db.get_task(connection, record["task_id"]) + current_run_id = ( + canonical_run_id(_task_value(task, "current_run_id", None)) + if task + else None + ) + current = str(current_run_id or "") + status = str(_task_value(task, "status", "") or "") if task else "" + finally: + connection.close() + if current != record["run_id"] or status != "running": + pool.store.finalize(record, "stale") + + +def _adopt_task( + pool: Any, + kanban_db: Any, + connection: Any, + board: str, + task: Any, + ordinals: list[int], + known: set[tuple[str, str, str]], +) -> None: + """Adopt one orphaned pool-eligible run onto a free ordinal.""" + task_id = str(_task_value(task, "id", "") or "") + raw_run_id = canonical_run_id(_task_value(task, "current_run_id", None)) + run_id = str(raw_run_id or "") + assignee = str(_task_value(task, "assignee", "") or "") + if ( + not task_id + or not run_id + or str(_task_value(task, "status", "")) != "running" + or not assignee.startswith("cli-") + or not distributed_workspace_eligible(task) + or (board, task_id, run_id) in known + ): + return + try: + payload = pool.assignment_payload(kanban_db, connection, task, board) + except RECOVERABLE_BOARD_ERRORS as error: + _defer(f"{board}/{task_id} recovery payload", error) + return + except Exception as error: # noqa: BLE001 - a real capability failure of this run + _block_exact( + kanban_db, board, task_id, raw_run_id, + "Distributed assignment recovery failed: " + f"{type(error).__name__}: {error}", + ) + return + binding = { + "board": board, "task_id": task_id, "run_id": run_id, + "worker_ordinal": ordinals[0], "attempt": 1, + } + try: + pool.store.add(binding, payload) + except ProtocolError as error: + # A durable row already owns this identity or ordinal. Leave the ordinal + # free and leave Kanban untouched; a later pass revisits it. + _defer(f"{board}/{task_id} adoption", error) + return + known.add((board, task_id, run_id)) + ordinals.pop(0) + + +def _adopt_board( + pool: Any, + kanban_db: Any, + board: str, + ordinals: list[int], + known: set[tuple[str, str, str]], +) -> None: + """Adopt this board's orphaned runs without touching any other board.""" + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + for task in kanban_db.list_tasks(connection): + if not ordinals: + return + _adopt_task(pool, kanban_db, connection, board, task, ordinals, known) + finally: + connection.close() + + +def reconcile(pool: Any) -> None: + """Recover the narrow claim/assignment crash gaps without double execution.""" + from hermes_cli import kanban_db + + for record in pool.store.active_assignments(): + try: + _release_moved_run(pool, kanban_db, record) + except Exception as error: # noqa: BLE001 - other assignments still reconcile + _defer(f"{_context(record)} release", error) + ordinals = pool.store.available_ordinals() + if not ordinals: + return + known = pool.store.known_runs() + try: + boards = kanban_db.list_boards(include_archived=False) + except RECOVERABLE_BOARD_ERRORS as error: + _defer("board-registry", error) + return + _settled("board-registry") + for raw_board in boards: + board = cli_lane_dispatch._board_slug(raw_board) + if not board or not ordinals: + continue + try: + _adopt_board(pool, kanban_db, board, ordinals, known) + _settled(f"{board} adoption") + except Exception as error: # noqa: BLE001 - later boards must still run + _defer(f"{board} adoption", error) + + +def _materialize( + pool: Any, + kanban_db: Any, + board: str, + task_id: str, + ordinal: int, + known: set[tuple[str, str, str]], +) -> None: + """Turn one freshly claimed run into exactly one durable assignment.""" + database_run_id: int | None = None + try: + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + task = kanban_db.get_task(connection, task_id) + if task is None: + return + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if not distributed_workspace_eligible(task): + if raw_run is not None: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed claim fenced because an existing " + "workspace is owned by the local lane" + ), + kind="capability", expected_run_id=raw_run, + ) + return + if raw_run is None: + raise ProjectPolicyError("claimed task has no canonical run ID") + database_run_id = raw_run + payload = pool.assignment_payload(kanban_db, connection, task, board) + finally: + connection.close() + run_id = str(database_run_id) + if (board, task_id, run_id) in known: + return + pool.store.add( + { + "board": board, "task_id": task_id, "run_id": run_id, + "worker_ordinal": ordinal, "attempt": 1, + }, + payload, + ) + known.add((board, task_id, run_id)) + _settled(f"{board}/{task_id} dispatch") + except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error: + # Coordinator-side storage/identity conditions. Never convert these into + # a Kanban capability block; reconcile() adopts the claim on a later pass. + _defer(f"{board}/{task_id} dispatch", error) + except Exception as error: # noqa: BLE001 - a real capability failure of this run + _block_exact( + kanban_db, board, task_id, database_run_id, + "Distributed assignment preparation failed: " + f"{type(error).__name__}: {error}", + ) + + +def dispatch(pool: Any) -> None: + """Claim up to the free ordinal count and materialize assignments.""" + from hermes_cli import kanban_db + + ordinals = pool.store.available_ordinals() + if not ordinals: + return + try: + # Only storage faults are absorbed here. An incompatible claim API must + # still surface, because silently skipping the eligibility predicate is + # how the pool would start claiming the local lane's owned workspaces. + claimed = cli_lane_dispatch.claim_ready( + set(), len(ordinals), + lambda _board, task: distributed_workspace_eligible(task), + ) + except RECOVERABLE_BOARD_ERRORS as error: + _defer("claim-ready", error) + return + _settled("claim-ready") + known = pool.store.known_runs() + for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False): + _materialize(pool, kanban_db, board, task_id, ordinal, known) diff --git a/services/hermes/scripts/execution_pool_protocol.py b/services/hermes/scripts/execution_pool_protocol.py index 599279be..007b8b0f 100644 --- a/services/hermes/scripts/execution_pool_protocol.py +++ b/services/hermes/scripts/execution_pool_protocol.py @@ -8,7 +8,6 @@ import hmac import json import os import re -import sqlite3 import stat import threading import time @@ -237,260 +236,15 @@ def parse_wire(body: bytes) -> dict[str, Any]: return value -class PoolStore: - """Coordinator-owned durable assignments; never stores provider secrets.""" +def __getattr__(name: str) -> Any: + """Expose the durable store on the stable protocol surface. - def __init__(self, path: Path, lease_seconds: int = 90): - self.path = path - self.lease_seconds = max(60, min(int(lease_seconds), 600)) - self._lock = threading.RLock() - path.parent.mkdir(parents=True, exist_ok=True) - self._initialize() + ``PoolStore`` moved to :mod:`execution_pool_store` to keep both modules under + the managed line ceiling. It is resolved lazily here so the store can keep + importing this module's primitives without an import cycle. + """ + if name == "PoolStore": + from execution_pool_store import PoolStore - def _connect(self) -> sqlite3.Connection: - connection = sqlite3.connect(self.path, timeout=10, isolation_level=None) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA synchronous=FULL") - connection.execute("PRAGMA busy_timeout=10000") - return connection - - def _initialize(self) -> None: - with self._connect() as connection: - connection.executescript( - """ - CREATE TABLE IF NOT EXISTS assignments ( - board TEXT NOT NULL, task_id TEXT NOT NULL, run_id TEXT NOT NULL, - worker_ordinal INTEGER NOT NULL CHECK(worker_ordinal BETWEEN 0 AND 2), - attempt INTEGER NOT NULL, assignment_digest TEXT NOT NULL, - payload_json TEXT NOT NULL, state TEXT NOT NULL, - lease_until REAL NOT NULL DEFAULT 0, last_heartbeat REAL NOT NULL DEFAULT 0, - result_digest TEXT, result_json TEXT, created_at REAL NOT NULL, - updated_at REAL NOT NULL, PRIMARY KEY(board, task_id, run_id) - ); - CREATE UNIQUE INDEX IF NOT EXISTS one_live_assignment_per_worker - ON assignments(worker_ordinal) WHERE state IN ('assigned','running','result'); - CREATE TABLE IF NOT EXISTS deliveries ( - delivery_id TEXT PRIMARY KEY, kind TEXT NOT NULL, digest TEXT NOT NULL, - received_at REAL NOT NULL - ); - """ - ) - - @staticmethod - def _record(row: sqlite3.Row | None) -> dict[str, Any] | None: - if row is None: - return None - value = dict(row) - value["payload"] = json.loads(value.pop("payload_json")) - if value.get("result_json"): - value["result"] = json.loads(value["result_json"]) - return value - - def add(self, binding: dict[str, Any], payload: dict[str, Any]) -> bool: - """Create exactly one assignment for a claimed run and free ordinal.""" - now = time.time() - digest = payload_digest(payload) - values = ( - binding["board"], binding["task_id"], binding["run_id"], - binding["worker_ordinal"], binding["attempt"], digest, - canonical_json(payload).decode(), "assigned", now, now, - ) - with self._lock, self._connect() as connection: - try: - connection.execute("BEGIN IMMEDIATE") - connection.execute( - """INSERT INTO assignments - (board,task_id,run_id,worker_ordinal,attempt,assignment_digest, - payload_json,state,created_at,updated_at) - VALUES (?,?,?,?,?,?,?,?,?,?)""", - values, - ) - connection.commit() - return True - except sqlite3.IntegrityError as error: - connection.rollback() - existing = connection.execute( - """SELECT assignment_digest,worker_ordinal,attempt FROM assignments - WHERE board=? AND task_id=? AND run_id=?""", - values[:3], - ).fetchone() - if existing and tuple(existing) == ( - digest, binding["worker_ordinal"], binding["attempt"] - ): - return False - if existing: - raise ProtocolError("conflicting duplicate assignment") from error - raise ProtocolError("worker ordinal already has a live assignment") from error - - def available_ordinals(self) -> list[int]: - with self._connect() as connection: - rows = connection.execute( - "SELECT worker_ordinal FROM assignments WHERE state IN ('assigned','running','result')" - ).fetchall() - occupied = {int(row[0]) for row in rows} - return [ordinal for ordinal in range(3) if ordinal not in occupied] - - def active_assignments(self) -> list[dict[str, Any]]: - with self._connect() as connection: - rows = connection.execute( - "SELECT * FROM assignments WHERE state IN ('assigned','running','result')" - ).fetchall() - return [self._record(row) or {} for row in rows] - - def offer(self, ordinal: int) -> dict[str, Any] | None: - """Return the ordinal's durable assignment, preserving restart identity.""" - now = time.time() - with self._lock, self._connect() as connection: - connection.execute("BEGIN IMMEDIATE") - row = connection.execute( - """SELECT * FROM assignments WHERE worker_ordinal=? - AND state IN ('assigned','running') ORDER BY created_at LIMIT 1""", - (ordinal,), - ).fetchone() - if row is not None: - connection.execute( - """UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? - WHERE board=? AND task_id=? AND run_id=?""", - (now + self.lease_seconds, now, now, row["board"], row["task_id"], row["run_id"]), - ) - connection.commit() - return self._record(row) - - def _matching(self, connection: sqlite3.Connection, envelope: dict[str, Any]) -> sqlite3.Row: - row = connection.execute( - "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", - (envelope["board"], envelope["task_id"], envelope["run_id"]), - ).fetchone() - if row is None: - raise ProtocolError("assignment is unknown or stale") - if int(row["worker_ordinal"]) != int(envelope["worker_ordinal"]): - raise ProtocolError("worker ordinal does not own this assignment") - if int(row["attempt"]) != int(envelope["attempt"]): - raise ProtocolError("assignment attempt is stale") - return row - - def heartbeat(self, envelope: dict[str, Any]) -> tuple[bool, bool]: - now = time.time() - delivery_digest = payload_digest( - { - name: envelope[name] - for name in ( - "kind", "board", "task_id", "run_id", "worker_ordinal", - "attempt", "payload_digest", - ) - } - ) - with self._lock, self._connect() as connection: - connection.execute("BEGIN IMMEDIATE") - duplicate = connection.execute( - "SELECT digest FROM deliveries WHERE delivery_id=?", - (envelope["delivery_id"],), - ).fetchone() - row = self._matching(connection, envelope) - if row["state"] not in {"assigned", "running"}: - raise ProtocolError("assignment is no longer running") - if row["state"] == "running" and float(row["lease_until"]) < now: - raise ProtocolError("assignment lease expired") - if duplicate and duplicate[0] != delivery_digest: - raise ProtocolError("delivery identifier was reused") - if not duplicate: - connection.execute( - "INSERT INTO deliveries VALUES (?,?,?,?)", - (envelope["delivery_id"], "heartbeat", delivery_digest, now), - ) - connection.execute( - "UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?", - (now + self.lease_seconds, now, now, envelope["board"], envelope["task_id"], envelope["run_id"]), - ) - connection.commit() - return True, bool(duplicate) - - def accept_result(self, envelope: dict[str, Any]) -> tuple[dict[str, Any], bool]: - now = time.time() - with self._lock, self._connect() as connection: - connection.execute("BEGIN IMMEDIATE") - row = self._matching(connection, envelope) - digest = envelope["payload_digest"] - if row["result_digest"]: - if row["result_digest"] != digest: - raise ProtocolError("conflicting result for completed delivery") - connection.rollback() - return self._record(row) or {}, True - if row["state"] not in {"assigned", "running"}: - raise ProtocolError("assignment cannot accept a result") - if row["state"] == "running" and float(row["lease_until"]) < now: - raise ProtocolError("assignment lease expired") - connection.execute( - """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=? - WHERE board=? AND task_id=? AND run_id=?""", - (digest, canonical_json(envelope["payload"]).decode(), now, - envelope["board"], envelope["task_id"], envelope["run_id"]), - ) - connection.commit() - row = connection.execute( - "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", - (envelope["board"], envelope["task_id"], envelope["run_id"]), - ).fetchone() - return self._record(row) or {}, False - - def pending_results(self) -> list[dict[str, Any]]: - with self._connect() as connection: - rows = connection.execute("SELECT * FROM assignments WHERE state='result' ORDER BY updated_at").fetchall() - return [self._record(row) or {} for row in rows] - - def expire_leases( - self, *, now: float | None = None, max_attempts: int = 3 - ) -> list[dict[str, Any]]: - """Fence expired attempts and re-offer or terminally release their ordinals.""" - current = time.time() if now is None else float(now) - maximum = max(1, min(int(max_attempts), 10)) - changed: list[dict[str, Any]] = [] - with self._lock, self._connect() as connection: - connection.execute("BEGIN IMMEDIATE") - rows = connection.execute( - """SELECT * FROM assignments WHERE state='running' - AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""", - (current,), - ).fetchall() - for row in rows: - if int(row["attempt"]) >= maximum: - state, attempt = "lease_failed", int(row["attempt"]) - else: - state, attempt = "assigned", int(row["attempt"]) + 1 - connection.execute( - """UPDATE assignments SET state=?,attempt=?,lease_until=0, - last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=? - AND attempt=? AND state='running'""", - ( - state, attempt, current, row["board"], row["task_id"], - row["run_id"], row["attempt"], - ), - ) - updated = connection.execute( - "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", - (row["board"], row["task_id"], row["run_id"]), - ).fetchone() - if updated is not None: - changed.append(self._record(updated) or {}) - connection.commit() - return changed - - def finalize(self, binding: dict[str, Any], state: str) -> None: - if state not in {"finalized", "stale"}: - raise ProtocolError("invalid terminal assignment state") - with self._lock, self._connect() as connection: - connection.execute( - "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?", - (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))), - ) - - def garbage_collect(self, retention_seconds: int) -> int: - cutoff = time.time() - max(3600, retention_seconds) - with self._lock, self._connect() as connection: - cursor = connection.execute( - "DELETE FROM assignments WHERE state IN ('finalized','stale') AND updated_at < ?", - (cutoff,), - ) - connection.execute("DELETE FROM deliveries WHERE received_at < ?", (cutoff,)) - return int(cursor.rowcount) + return PoolStore + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/services/hermes/scripts/execution_pool_scm.py b/services/hermes/scripts/execution_pool_scm.py index 6badfbb9..393031db 100644 --- a/services/hermes/scripts/execution_pool_scm.py +++ b/services/hermes/scripts/execution_pool_scm.py @@ -144,6 +144,46 @@ def _state_path(envelope: dict[str, Any]) -> Path: return path +def submission_refs(branch: str, attempt: int, head: str) -> tuple[str, ...]: + """Candidate push refs for one attempt, in order of preference. + + The broker accepts branch *creation* only, so a retry that adds commits can + never update the ref a previous attempt already published, and no attempt may + ever move a protected ref. Each fallback is therefore a fresh name in the same + reviewed namespace -- bound first to the exact attempt, then to the exact + content -- so every push stays a creation and stays idempotent under replay. + """ + candidates = ( + branch, + f"{branch}-attempt-{max(1, int(attempt))}", + f"{branch}-{head[:12]}", + ) + allowed: list[str] = [] + for candidate in dict.fromkeys(candidates): + try: + allowed.append(validate_branch(candidate, feature=True)) + except ValueError: + continue + if not allowed: + raise ProtocolError("no reviewed branch name is available for this attempt") + return tuple(allowed) + + +def _remote_heads(destination: Path, refs: tuple[str, ...]) -> dict[str, str]: + """Read the broker's current head for each candidate ref, read-only.""" + output = _run( + "ls-remote", "--heads", "hermes-broker", + *(f"refs/heads/{ref}" for ref in refs), + cwd=destination, timeout=300, + ) + heads: dict[str, str] = {} + for line in output.splitlines(): + fields = line.split() + if len(fields) == 2 and fields[1].startswith("refs/heads/"): + heads[fields[1].removeprefix("refs/heads/")] = fields[0] + return heads + + def _workspace_identity(destination: Path, repo: str, branch: str) -> str: """Validate the checkout using bounded Git plumbing with all hooks disabled.""" if destination.is_symlink() or not (destination / ".git").is_dir(): @@ -270,12 +310,26 @@ class Boundary: ) if status: raise ProtocolError("workspace has uncommitted or untracked files") - ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination)) - if ahead <= 0: - return {"workspace": str(destination), "branch": branch, "pull_request": ""} - _run( - "push", "hermes-broker", f"HEAD:refs/heads/{branch}", - cwd=destination, timeout=900, + candidates = submission_refs(branch, int(envelope["attempt"]), head) + heads = _remote_heads(destination, candidates) + # A candidate already at this exact head was published by an earlier + # attempt or an interrupted one: adopt it instead of pushing again, so + # replay is a no-op and prior work is never re-derived or dropped. + published = next( + (ref for ref in candidates if heads.get(ref) == head), None ) - pull = self._draft(repo, branch, base, head, title, body) - return {"workspace": str(destination), "branch": branch, "pull_request": pull} + ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination)) + if ahead <= 0 and published is None: + return {"workspace": str(destination), "branch": branch, "pull_request": ""} + target = published or next( + (ref for ref in candidates if ref not in heads), None + ) + if target is None: + raise ProtocolError("every reviewed branch name for this attempt is taken") + if published is None: + _run( + "push", "hermes-broker", f"HEAD:refs/heads/{target}", + cwd=destination, timeout=900, + ) + pull = self._draft(repo, target, base, head, title, body) + return {"workspace": str(destination), "branch": target, "pull_request": pull} diff --git a/services/hermes/scripts/execution_pool_server.py b/services/hermes/scripts/execution_pool_server.py index b2a01713..75e08b64 100644 --- a/services/hermes/scripts/execution_pool_server.py +++ b/services/hermes/scripts/execution_pool_server.py @@ -9,6 +9,7 @@ import sqlite3 import sys import threading import time +from collections.abc import Iterable from http.server import BaseHTTPRequestHandler from pathlib import Path from typing import Any @@ -17,12 +18,12 @@ from execution_pool_protocol import ( MAX_WIRE_BYTES, PROTOCOL_VERSION, BoundedHTTPServer, - PoolStore, ProtocolError, canonical_json, parse_wire, read_key, ) +from execution_pool_store import PoolStore DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) @@ -92,19 +93,41 @@ def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]: return Handler +def maintenance_cycle(operations: Iterable[Any]) -> int: + """Run one maintenance pass; a failing operation never aborts the others. + + Startup uses this too. A durable row that one pass cannot process must not + stop the coordinator from binding its port, because the store lives on a PVC + and would otherwise turn a single bad row into a permanent crash loop. + """ + deferred = 0 + for operation in operations: + try: + operation() + except Exception as error: # noqa: BLE001 - the loop is the recovery mechanism + deferred += 1 + print( + f"pool maintenance deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + return deferred + + def run(coordinator_type: type[Any]) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--once", action="store_true") args = parser.parse_args() store = PoolStore(STATE_ROOT / "assignments.db") coordinator = coordinator_type(read_key(KEY_PATH), store) - for operation in ( + operations = ( coordinator.expire_leases, coordinator.recover_results, coordinator.reconcile, coordinator.dispatch, - ): - operation() + lambda: store.garbage_collect(RETENTION_SECONDS), + ) + maintenance_cycle(operations) if args.once: return 0 server = BoundedHTTPServer( @@ -112,19 +135,5 @@ def run(coordinator_type: type[Any]) -> int: ) threading.Thread(target=server.serve_forever, daemon=True).start() while True: - for operation in ( - coordinator.expire_leases, - coordinator.recover_results, - coordinator.reconcile, - coordinator.dispatch, - lambda: store.garbage_collect(RETENTION_SECONDS), - ): - try: - operation() - except Exception as error: - print( - f"pool maintenance deferred: {type(error).__name__}: {error}", - file=sys.stderr, - flush=True, - ) + maintenance_cycle(operations) time.sleep(5) diff --git a/services/hermes/scripts/execution_pool_store.py b/services/hermes/scripts/execution_pool_store.py new file mode 100644 index 00000000..1022f78a --- /dev/null +++ b/services/hermes/scripts/execution_pool_store.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Durable, restart-safe assignment state for the Hermes execution pool. + +The store owns every transition of a claimed Kanban run through the pool. It +deliberately keeps two invariants that the coordinator depends on for recovery: + +* ``lease_failed`` is a *retryable* state, not a terminal one. A row reaches it + when the pool has released the ordinal but has not yet confirmed the outcome + with Kanban, so the coordinator must keep retrying it. +* Only ``finalized`` and ``stale`` are terminal, and only terminal rows are ever + garbage-collected. A row is therefore never removed before the pool holds + authoritative evidence about what happened to its run. +""" + +from __future__ import annotations + +import json +import sqlite3 +import threading +import time +from pathlib import Path +from typing import Any + +from execution_pool_protocol import ProtocolError, canonical_json, payload_digest + + +LIVE_STATES = ("assigned", "running", "result") +TERMINAL_STATES = ("finalized", "stale") +LEASE_FAILED = "lease_failed" +SCHEMA = """ +CREATE TABLE IF NOT EXISTS assignments ( + board TEXT NOT NULL, task_id TEXT NOT NULL, run_id TEXT NOT NULL, + worker_ordinal INTEGER NOT NULL CHECK(worker_ordinal BETWEEN 0 AND 2), + attempt INTEGER NOT NULL, assignment_digest TEXT NOT NULL, + payload_json TEXT NOT NULL, state TEXT NOT NULL, + lease_until REAL NOT NULL DEFAULT 0, last_heartbeat REAL NOT NULL DEFAULT 0, + result_digest TEXT, result_json TEXT, created_at REAL NOT NULL, + updated_at REAL NOT NULL, PRIMARY KEY(board, task_id, run_id) +); +CREATE UNIQUE INDEX IF NOT EXISTS one_live_assignment_per_worker + ON assignments(worker_ordinal) WHERE state IN ('assigned','running','result'); +CREATE TABLE IF NOT EXISTS deliveries ( + delivery_id TEXT PRIMARY KEY, kind TEXT NOT NULL, digest TEXT NOT NULL, + received_at REAL NOT NULL +); +""" + + +class PoolStore: + """Coordinator-owned durable assignments; never stores provider secrets.""" + + def __init__(self, path: Path, lease_seconds: int = 90): + self.path = path + self.lease_seconds = max(60, min(int(lease_seconds), 600)) + self._lock = threading.RLock() + path.parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, timeout=10, isolation_level=None) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA busy_timeout=10000") + return connection + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript(SCHEMA) + + @staticmethod + def _record(row: sqlite3.Row | None) -> dict[str, Any] | None: + if row is None: + return None + value = dict(row) + value["payload"] = json.loads(value.pop("payload_json")) + if value.get("result_json"): + value["result"] = json.loads(value["result_json"]) + return value + + def _select(self, sql: str) -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute(sql).fetchall() + return [self._record(row) or {} for row in rows] + + def add(self, binding: dict[str, Any], payload: dict[str, Any]) -> bool: + """Create exactly one assignment for a claimed run and free ordinal.""" + now = time.time() + digest = payload_digest(payload) + values = ( + binding["board"], binding["task_id"], binding["run_id"], + binding["worker_ordinal"], binding["attempt"], digest, + canonical_json(payload).decode(), "assigned", now, now, + ) + with self._lock, self._connect() as connection: + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + """INSERT INTO assignments + (board,task_id,run_id,worker_ordinal,attempt,assignment_digest, + payload_json,state,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + values, + ) + connection.commit() + return True + except sqlite3.IntegrityError as error: + connection.rollback() + existing = connection.execute( + """SELECT assignment_digest,worker_ordinal,attempt FROM assignments + WHERE board=? AND task_id=? AND run_id=?""", + values[:3], + ).fetchone() + if existing and tuple(existing) == ( + digest, binding["worker_ordinal"], binding["attempt"] + ): + return False + if existing: + raise ProtocolError("conflicting duplicate assignment") from error + raise ProtocolError("worker ordinal already has a live assignment") from error + + def available_ordinals(self) -> list[int]: + with self._connect() as connection: + rows = connection.execute( + "SELECT worker_ordinal FROM assignments WHERE state IN ('assigned','running','result')" + ).fetchall() + occupied = {int(row[0]) for row in rows} + return [ordinal for ordinal in range(3) if ordinal not in occupied] + + def active_assignments(self) -> list[dict[str, Any]]: + return self._select( + "SELECT * FROM assignments WHERE state IN ('assigned','running','result')" + ) + + def known_runs(self) -> set[tuple[str, str, str]]: + """Every run identity this store already owns a row for, in any state. + + Adoption must consult this rather than only the live states: re-adding a + run that still has a retrying or not-yet-collected row would collide with + ``PRIMARY KEY(board, task_id, run_id)`` under a different payload digest. + """ + with self._connect() as connection: + rows = connection.execute( + "SELECT board,task_id,run_id FROM assignments" + ).fetchall() + return {(str(row[0]), str(row[1]), str(row[2])) for row in rows} + + def offer(self, ordinal: int) -> dict[str, Any] | None: + """Return the ordinal's durable assignment, preserving restart identity.""" + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + """SELECT * FROM assignments WHERE worker_ordinal=? + AND state IN ('assigned','running') ORDER BY created_at LIMIT 1""", + (ordinal,), + ).fetchone() + if row is not None: + connection.execute( + """UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (now + self.lease_seconds, now, now, row["board"], row["task_id"], row["run_id"]), + ) + connection.commit() + return self._record(row) + + def _matching(self, connection: sqlite3.Connection, envelope: dict[str, Any]) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + if row is None: + raise ProtocolError("assignment is unknown or stale") + if int(row["worker_ordinal"]) != int(envelope["worker_ordinal"]): + raise ProtocolError("worker ordinal does not own this assignment") + if int(row["attempt"]) != int(envelope["attempt"]): + raise ProtocolError("assignment attempt is stale") + return row + + def heartbeat(self, envelope: dict[str, Any]) -> tuple[bool, bool]: + now = time.time() + delivery_digest = payload_digest( + { + name: envelope[name] + for name in ( + "kind", "board", "task_id", "run_id", "worker_ordinal", + "attempt", "payload_digest", + ) + } + ) + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + duplicate = connection.execute( + "SELECT digest FROM deliveries WHERE delivery_id=?", + (envelope["delivery_id"],), + ).fetchone() + row = self._matching(connection, envelope) + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment is no longer running") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + if duplicate and duplicate[0] != delivery_digest: + raise ProtocolError("delivery identifier was reused") + if not duplicate: + connection.execute( + "INSERT INTO deliveries VALUES (?,?,?,?)", + (envelope["delivery_id"], "heartbeat", delivery_digest, now), + ) + connection.execute( + "UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?", + (now + self.lease_seconds, now, now, envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + return True, bool(duplicate) + + def accept_result(self, envelope: dict[str, Any]) -> tuple[dict[str, Any], bool]: + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = self._matching(connection, envelope) + digest = envelope["payload_digest"] + if row["result_digest"]: + if row["result_digest"] != digest: + raise ProtocolError("conflicting result for completed delivery") + connection.rollback() + return self._record(row) or {}, True + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment cannot accept a result") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + connection.execute( + """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (digest, canonical_json(envelope["payload"]).decode(), now, + envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + return self._record(row) or {}, False + + def pending_results(self) -> list[dict[str, Any]]: + return self._select( + "SELECT * FROM assignments WHERE state='result' ORDER BY updated_at" + ) + + def failed_leases(self) -> list[dict[str, Any]]: + """Rows whose ordinal is released but whose Kanban outcome is unconfirmed. + + The coordinator drains this on every maintenance pass, so a Kanban write + that failed at expiry time is retried instead of stranding the run. + """ + return self._select( + "SELECT * FROM assignments WHERE state='lease_failed' ORDER BY updated_at" + ) + + def expire_leases( + self, *, now: float | None = None, max_attempts: int = 3 + ) -> list[dict[str, Any]]: + """Fence expired attempts and re-offer or terminally release their ordinals.""" + current = time.time() if now is None else float(now) + maximum = max(1, min(int(max_attempts), 10)) + changed: list[dict[str, Any]] = [] + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + rows = connection.execute( + """SELECT * FROM assignments WHERE state='running' + AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""", + (current,), + ).fetchall() + for row in rows: + if int(row["attempt"]) >= maximum: + state, attempt = LEASE_FAILED, int(row["attempt"]) + else: + state, attempt = "assigned", int(row["attempt"]) + 1 + connection.execute( + """UPDATE assignments SET state=?,attempt=?,lease_until=0, + last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=? + AND attempt=? AND state='running'""", + ( + state, attempt, current, row["board"], row["task_id"], + row["run_id"], row["attempt"], + ), + ) + updated = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (row["board"], row["task_id"], row["run_id"]), + ).fetchone() + if updated is not None: + changed.append(self._record(updated) or {}) + connection.commit() + return changed + + def finalize(self, binding: dict[str, Any], state: str) -> bool: + """Apply one terminal state to the exact run, ordinal, and attempt.""" + if state not in TERMINAL_STATES: + raise ProtocolError("invalid terminal assignment state") + with self._lock, self._connect() as connection: + cursor = connection.execute( + "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?", + (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))), + ) + return bool(cursor.rowcount) + + def garbage_collect(self, retention_seconds: int) -> int: + """Remove aged rows, and only ones with authoritative terminal evidence. + + ``lease_failed`` is excluded on purpose: dropping a row whose Kanban + outcome was never confirmed would lose the pool's only record that the + run still needs to be surfaced. + """ + cutoff = time.time() - max(3600, retention_seconds) + with self._lock, self._connect() as connection: + cursor = connection.execute( + "DELETE FROM assignments WHERE state IN ('finalized','stale') AND updated_at < ?", + (cutoff,), + ) + connection.execute("DELETE FROM deliveries WHERE received_at < ?", (cutoff,)) + return int(cursor.rowcount) diff --git a/testing/quality_contract.json b/testing/quality_contract.json index c5598914..1a3e10e2 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -29,10 +29,12 @@ "services/hermes/scm-common/scripts/scm_broker_server.py", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_maintenance.py", "services/hermes/scripts/execution_pool_project.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_store.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/hermes_coordinator.py", "services/hermes/scripts/node_account_audit.py", @@ -83,10 +85,12 @@ "services/hermes/scm-common/scripts", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_maintenance.py", "services/hermes/scripts/execution_pool_project.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_store.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/hermes_coordinator.py", "services/hermes/scripts/node_account_audit.py", @@ -262,10 +266,12 @@ "services/hermes/scripts/stage_runtime_access.py", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_maintenance.py", "services/hermes/scripts/execution_pool_project.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_store.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/cli_lane_board.py", "services/hermes/scripts/cli_lane_capabilities.py", @@ -302,10 +308,12 @@ "services/hermes/scm-common/scripts/scm_broker_server.py", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_maintenance.py", "services/hermes/scripts/execution_pool_project.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_store.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/hermes_coordinator.py", "services/hermes/scripts/node_account_audit.py", diff --git a/testing/tests/test_hermes_execution_pool.py b/testing/tests/test_hermes_execution_pool.py index 8b9966d5..c82df4d7 100644 --- a/testing/tests/test_hermes_execution_pool.py +++ b/testing/tests/test_hermes_execution_pool.py @@ -348,8 +348,17 @@ def test_three_node_statefulset_contract_and_cross_worker_isolation(): item["metadata"]["name"]: item for item in stateful["spec"]["volumeClaimTemplates"] } - assert set(claims) == {"workspace", "provider-access"} - assert all(item["spec"]["accessModes"] == ["ReadWriteOnce"] for item in claims.values()) + assert set(claims) == {"workspace", "tools", "provider-access"} + # The workspace is co-mounted by this ordinal's mediator, so it must be + # ReadWriteMany: a ReadWriteOnce claim deadlocks on Multi-Attach when a drain + # or preemption separates the two Pods. Everything only the worker mounts + # stays single-mounter. + assert claims["workspace"]["spec"]["accessModes"] == ["ReadWriteMany"] + assert claims["tools"]["spec"]["accessModes"] == ["ReadWriteOnce"] + assert claims["provider-access"]["spec"]["accessModes"] == ["ReadWriteOnce"] + assert all( + item["spec"]["storageClassName"] == "astreae" for item in claims.values() + ) anti = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] assert anti[0]["topologyKey"] == "kubernetes.io/hostname" spread = pod["topologySpreadConstraints"][0] diff --git a/testing/tests/test_hermes_execution_pool_dispatch_v2.py b/testing/tests/test_hermes_execution_pool_dispatch_v2.py index 57dd0f74..c63e9825 100644 --- a/testing/tests/test_hermes_execution_pool_dispatch_v2.py +++ b/testing/tests/test_hermes_execution_pool_dispatch_v2.py @@ -16,6 +16,7 @@ SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_maintenance as maintenance # noqa: E402 import execution_pool_protocol as protocol # noqa: E402 import execution_pool_server as server # noqa: E402 from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 @@ -84,7 +85,7 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch( observed.append((active, limit, eligible("metis", item))) return [("metis", item.id)] - monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", claim) + monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim) monkeypatch.setattr( coordinator, "assignment_payload", lambda *_a: assignment_payload(branch="wt/t_deadbeef"), @@ -109,7 +110,7 @@ def test_dispatch_incompatible_unversioned_claim_api_fails_closed( calls.append((active, limit)) return [("metis", owned.id)] - monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", old_claim) + monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", old_claim) with pytest.raises(TypeError): pool.dispatch() assert calls == [] @@ -125,7 +126,7 @@ def test_dispatch_workspace_ownership_race_is_exactly_fenced( store = protocol.PoolStore(tmp_path / "pool.db") pool = coordinator.Coordinator(MASTER, store) monkeypatch.setattr( - coordinator.cli_lane_dispatch, + maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", owned.id)], ) @@ -154,7 +155,7 @@ def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim( store = protocol.PoolStore(tmp_path / "pool.db") pool = coordinator.Coordinator(MASTER, store) monkeypatch.setattr( - coordinator.cli_lane_dispatch, + maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", broken.id)], ) @@ -170,7 +171,7 @@ def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim( ) pool = coordinator.Coordinator(MASTER, full) monkeypatch.setattr( - coordinator.cli_lane_dispatch, + maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: (_ for _ in ()).throw(AssertionError("must not claim")), ) @@ -285,7 +286,7 @@ def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths( empty = protocol.PoolStore(tmp_path / "missing.db") monkeypatch.setattr( - coordinator.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")] + maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")] ) coordinator.Coordinator(MASTER, empty).dispatch() diff --git a/testing/tests/test_hermes_execution_pool_isolation.py b/testing/tests/test_hermes_execution_pool_isolation.py new file mode 100644 index 00000000..7bac8be5 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_isolation.py @@ -0,0 +1,391 @@ +"""Fault isolation contracts for the execution-pool maintenance passes. + +No single poisoned row, task, or board may abort the work queued behind it, and a +coordinator-side fault must never be converted into a Kanban mutation. Startup +maintenance is held to exactly the same standard as the steady-state loop, +because the durable store lives on a PVC and would otherwise crash-loop. +""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_maintenance as maintenance # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_server as server # noqa: E402 +import execution_pool_store as pool_store # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + MASTER, + assignment_payload, + binding, + install_kanban, + task, +) +from testing.tests.test_hermes_execution_pool_support import ( # noqa: E402 + LOCKED, + clear_pool_deferrals, # noqa: F401 - autouse fixture, resolved by name + exhausted_store, + states, +) + + +def test_one_poisoned_board_and_task_never_abort_the_work_behind_them( + tmp_path, monkeypatch, capsys +): + first = task(id="t_first", current_run_id=41) + second = task(id="t_second", current_run_id=42) + kanban = install_kanban( + monkeypatch, [first, second], tmp_path, boards=["broken", "metis"] + ) + real_list_tasks = kanban.list_tasks + payloads = {"calls": 0} + + def flaky_list_tasks(connection): + if kanban.current == "broken": + raise LOCKED + return real_list_tasks(connection) + + def flaky_payload(_db, _connection, item, _board): + payloads["calls"] += 1 + if item.id == "t_first": + raise LOCKED + return assignment_payload() + + def scoped(board): + kanban.current = board + return kanban.scope(board) + + kanban.current = "" + kanban.scope = kanban.scoped_current_board + kanban.scoped_current_board = scoped + kanban.list_tasks = flaky_list_tasks + monkeypatch.setattr(coordinator, "assignment_payload", flaky_payload) + store = pool_store.PoolStore(tmp_path / "poison.db") + coordinator.Coordinator(MASTER, store).reconcile() + + assert payloads["calls"] == 2 + assert states(store) == [("42", 0, 1, "assigned")] + assert kanban.blocked == [] + captured = capsys.readouterr().err + assert "broken adoption" in captured + assert "t_first recovery payload" in captured + + +def test_adoption_conflict_leaves_the_ordinal_free_and_kanban_untouched( + tmp_path, monkeypatch, capsys +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + store = pool_store.PoolStore(tmp_path / "conflict.db") + pool = coordinator.Coordinator(MASTER, store) + + def conflict(_binding, _payload): + raise protocol.ProtocolError("conflicting duplicate assignment") + + monkeypatch.setattr(store, "add", conflict) + pool.reconcile() + assert store.available_ordinals() == [0, 1, 2] + assert kanban.blocked == [] + assert "t_deadbeef adoption" in capsys.readouterr().err + + +def test_dispatch_defers_storage_and_identity_faults_without_blocking_the_run( + tmp_path, monkeypatch, capsys +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: [("metis", live.id)], + ) + store = pool_store.PoolStore(tmp_path / "dispatch.db") + pool = coordinator.Coordinator(MASTER, store) + + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_args: (_ for _ in ()).throw(LOCKED), + ) + pool.dispatch() + assert kanban.blocked == [] + assert store.active_assignments() == [] + assert "metis/t_deadbeef dispatch" in capsys.readouterr().err + + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + monkeypatch.setattr( + store, "add", + lambda *_args: (_ for _ in ()).throw( + protocol.ProtocolError("worker ordinal already has a live assignment") + ), + ) + pool.dispatch() + assert kanban.blocked == [] + + +def test_dispatch_still_parks_the_exact_run_on_a_real_capability_failure( + tmp_path, monkeypatch +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: [("metis", live.id)], + ) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_args: (_ for _ in ()).throw(RuntimeError("context exceeds 32KiB")), + ) + store = pool_store.PoolStore(tmp_path / "capability.db") + coordinator.Coordinator(MASTER, store).dispatch() + assert kanban.blocked[-1][0] == "t_deadbeef" + assert kanban.blocked[-1][1]["kind"] == "capability" + assert kanban.blocked[-1][1]["expected_run_id"] == 23 + assert "32KiB" in kanban.blocked[-1][1]["reason"] + + +def test_a_failing_park_write_is_absorbed_rather_than_wedging_the_pass( + tmp_path, monkeypatch, capsys +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + kanban.block_task = lambda *_args, **_kwargs: (_ for _ in ()).throw(LOCKED) + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: [("metis", live.id)], + ) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_args: (_ for _ in ()).throw(RuntimeError("bad")), + ) + store = pool_store.PoolStore(tmp_path / "park.db") + coordinator.Coordinator(MASTER, store).dispatch() + assert "t_deadbeef block" in capsys.readouterr().err + + +def test_one_unprocessable_result_never_blocks_the_results_behind_it( + tmp_path, monkeypatch, capsys +): + install_kanban(monkeypatch, [task(), task(id="t_second", current_run_id=24)], tmp_path) + store = pool_store.PoolStore(tmp_path / "results.db") + pool = coordinator.Coordinator(MASTER, store) + handled = [] + + def finalize(record): + if record["task_id"] == "t_deadbeef": + raise protocol.ProtocolError("result payload must be an object") + handled.append(record["task_id"]) + + monkeypatch.setattr( + store, "pending_results", + lambda: [{**binding(), "result": []}, {**binding(task_id="t_second"), "result": {}}], + ) + monkeypatch.setattr(pool, "finalize", finalize) + pool.recover_results() + assert handled == ["t_second"] + assert "result recovery deferred" in capsys.readouterr().err + + +def test_startup_maintenance_is_as_safe_as_the_steady_state_loop( + tmp_path, monkeypatch, capsys +): + """A poisoned store must not stop the coordinator from binding its port.""" + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + kanban.block_task = lambda *_args, **_kwargs: (_ for _ in ()).throw(LOCKED) + kanban.list_boards = lambda include_archived=False: (_ for _ in ()).throw( + RuntimeError("board registry unavailable") + ) + key = tmp_path / "key" + key.write_bytes(b"m" * 48) + key.chmod(0o600) + monkeypatch.setattr(server, "STATE_ROOT", tmp_path / "state") + monkeypatch.setattr(server, "KEY_PATH", key) + monkeypatch.setattr(sys, "argv", ["execution_pool_server", "--once"]) + exhausted_store(tmp_path / "state/assignments.db") + + assert server.run(coordinator.Coordinator) == 0 + captured = capsys.readouterr().err + assert "pool maintenance deferred" in captured + row = sqlite3.connect(tmp_path / "state/assignments.db").execute( + "SELECT state FROM assignments" + ).fetchone() + assert row[0] == "lease_failed" + + +def test_maintenance_cycle_counts_deferrals_and_keeps_running(capsys): + calls = [] + count = server.maintenance_cycle( + ( + lambda: calls.append("first"), + lambda: (_ for _ in ()).throw(RuntimeError("boom")), + lambda: calls.append("last"), + ) + ) + assert count == 1 + assert calls == ["first", "last"] + assert "pool maintenance deferred: RuntimeError: boom" in capsys.readouterr().err + + +def test_the_poison_row_survives_restart_on_the_same_pvc_and_then_resolves( + tmp_path, monkeypatch, capsys +): + """The store lives on hermes-agent-home, so recovery must survive restarts.""" + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + kanban.block_task = lambda *_args, **_kwargs: (_ for _ in ()).throw(LOCKED) + database = tmp_path / "pvc/assignments.db" + first = exhausted_store(database) + coordinator.Coordinator(MASTER, first).expire_leases() + assert states(first) == [("23", 0, 3, "lease_failed")] + del first + capsys.readouterr() + + # A fresh process attaches the same PVC path and finds the row waiting. + restarted = pool_store.PoolStore(database) + assert restarted.failed_leases()[0]["state"] == "lease_failed" + assert restarted.available_ordinals() == [0, 1, 2] + kanban.block_task = lambda _connection, task_id, **values: ( + kanban.blocked.append((task_id, values)) or True + ) + coordinator.Coordinator(MASTER, restarted).expire_leases() + assert states(restarted) == [("23", 0, 3, "finalized")] + assert kanban.blocked[-1][1]["expected_run_id"] == 23 + + +def test_a_store_write_fault_while_recording_leaves_the_row_retryable( + tmp_path, monkeypatch, capsys +): + live = task() + install_kanban(monkeypatch, [live], tmp_path) + store = exhausted_store(tmp_path / "store-fault.db") + pool = coordinator.Coordinator(MASTER, store) + real_finalize = store.finalize + monkeypatch.setattr( + store, "finalize", lambda *_args: (_ for _ in ()).throw(LOCKED) + ) + pool.expire_leases() + assert states(store) == [("23", 0, 3, "lease_failed")] + assert "lease park" in capsys.readouterr().err + + monkeypatch.setattr(store, "finalize", real_finalize) + pool.expire_leases() + assert states(store) == [("23", 0, 3, "finalized")] + + +def test_finalize_reports_whether_the_exact_row_changed(tmp_path): + store = pool_store.PoolStore(tmp_path / "finalize.db") + store.add(binding(), assignment_payload()) + assert store.finalize(binding(attempt=9), "stale") is False + assert store.finalize(binding(), "stale") is True + with pytest.raises(protocol.ProtocolError, match="terminal"): + store.finalize(binding(), "lease_failed") + + +def test_deferral_reporting_is_deduplicated_and_bounded(capsys): + maintenance._defer("ctx", LOCKED) + maintenance._defer("ctx", LOCKED) + assert capsys.readouterr().err.count("pool work deferred") == 1 + maintenance._defer("ctx", RuntimeError("changed")) + assert "changed" in capsys.readouterr().err + for index in range(maintenance.MAX_DEFERRALS + 1): + maintenance._defer(f"ctx-{index}", LOCKED) + assert len(maintenance.DEFERRALS) <= maintenance.MAX_DEFERRALS + capsys.readouterr() + + +def test_protocol_exposes_only_the_store_on_its_compatibility_surface(): + assert protocol.PoolStore is pool_store.PoolStore + missing = "SomethingElse" + with pytest.raises(AttributeError, match="no attribute"): + getattr(protocol, missing) + + +def test_adoption_stops_as_soon_as_the_last_ordinal_is_taken(tmp_path, monkeypatch): + items = [task(id=f"t_{index}", current_run_id=50 + index) for index in range(4)] + kanban = install_kanban(monkeypatch, items, tmp_path) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + store = pool_store.PoolStore(tmp_path / "full.db") + coordinator.Coordinator(MASTER, store).reconcile() + assert len(store.active_assignments()) == 3 + assert store.available_ordinals() == [] + assert kanban.blocked == [] + + +def test_an_unreadable_board_registry_defers_instead_of_escaping( + tmp_path, monkeypatch, capsys +): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + kanban.list_boards = lambda include_archived=False: (_ for _ in ()).throw(LOCKED) + store = pool_store.PoolStore(tmp_path / "registry.db") + coordinator.Coordinator(MASTER, store).reconcile() + assert store.active_assignments() == [] + assert "board-registry" in capsys.readouterr().err + + +def test_an_unreadable_claim_path_defers_instead_of_escaping( + tmp_path, monkeypatch, capsys +): + install_kanban(monkeypatch, [task()], tmp_path) + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: (_ for _ in ()).throw(LOCKED), + ) + store = pool_store.PoolStore(tmp_path / "claim.db") + coordinator.Coordinator(MASTER, store).dispatch() + assert store.active_assignments() == [] + assert "claim-ready" in capsys.readouterr().err + + +def test_an_owned_workspace_without_a_canonical_run_is_skipped_silently( + tmp_path, monkeypatch +): + owned = task(workspace_path="/owned", current_run_id=None) + kanban = install_kanban(monkeypatch, [owned], tmp_path) + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: [("metis", owned.id)], + ) + store = pool_store.PoolStore(tmp_path / "owned.db") + coordinator.Coordinator(MASTER, store).dispatch() + assert kanban.blocked == [] + assert store.active_assignments() == [] + + +def test_a_result_cannot_be_accepted_onto_an_already_terminal_row(tmp_path): + store = pool_store.PoolStore(tmp_path / "terminal.db") + store.add(binding(), assignment_payload()) + store.finalize(binding(), "finalized") + envelope = {**binding(), "payload_digest": "d" * 64, "payload": {}} + with pytest.raises(protocol.ProtocolError, match="cannot accept a result"): + store.accept_result(envelope) + + +def test_a_nonstring_message_kind_is_rejected_before_any_lookup(): + envelope = { + "version": protocol.PROTOCOL_VERSION, "kind": 7, "board": "metis", + "task_id": "t_deadbeef", "run_id": "23", "worker_ordinal": 0, "attempt": 1, + "delivery_id": "d1", "issued_at": 0, "expires_at": 1, + "payload_digest": "x", "payload": {}, "signature": "s", + } + with pytest.raises(protocol.ProtocolError, match="unexpected message kind"): + protocol.verify_envelope(b"k" * 32, envelope) diff --git a/testing/tests/test_hermes_execution_pool_manifests.py b/testing/tests/test_hermes_execution_pool_manifests.py new file mode 100644 index 00000000..8530e0ec --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_manifests.py @@ -0,0 +1,83 @@ +"""Shared manifest and scheduling helpers for the execution-pool suites.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services/hermes" +FLUX = ROOT / "clusters/atlas/flux-system/applications" + + +def documents(path): + """Every non-empty YAML document in one manifest file.""" + return [item for item in yaml.safe_load_all(path.read_text()) if item] + + +def worker(): + """The rendered-source hermes-execution-worker StatefulSet.""" + return documents(HERMES / "execution-worker-statefulset.yaml")[0] + + +def mediators(): + """The three per-ordinal mediator Deployments, in ordinal order.""" + return [ + item + for item in documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + + +def node(name, *, ready=True, cordoned=False): + """One simulated pool-eligible worker node.""" + return { + "name": name, "ready": ready, "cordoned": cordoned, + "labels": { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true", + "kubernetes.io/hostname": name, + }, + } + + +def matches(candidate, expressions): + """Whether one node satisfies a nodeAffinity match-expression list.""" + for expression in expressions: + actual = candidate["labels"].get(expression["key"]) + values = expression.get("values", []) + if expression["operator"] == "In" and actual not in values: + return False + if expression["operator"] == "NotIn" and actual in values: + return False + return True + + +def placeable(pod, nodes, *, occupied=()): + """Nodes this Pod spec could be scheduled onto, honouring hard constraints.""" + affinity = pod.get("affinity", {}) + required = affinity.get("nodeAffinity", {}).get( + "requiredDuringSchedulingIgnoredDuringExecution", {} + ) + terms = required.get("nodeSelectorTerms", [{"matchExpressions": []}]) + hard_pod = affinity.get("podAffinity", {}).get( + "requiredDuringSchedulingIgnoredDuringExecution", [] + ) + anti = affinity.get("podAntiAffinity", {}).get( + "requiredDuringSchedulingIgnoredDuringExecution", [] + ) + allowed = [] + for candidate in nodes: + if not candidate["ready"] or candidate["cordoned"]: + continue + if not any(matches(candidate, term["matchExpressions"]) for term in terms): + continue + if anti and candidate["name"] in occupied: + continue + if hard_pod: + # A hard podAffinity can only be satisfied where its target already is. + continue + allowed.append(candidate["name"]) + return allowed diff --git a/testing/tests/test_hermes_execution_pool_placement.py b/testing/tests/test_hermes_execution_pool_placement.py new file mode 100644 index 00000000..906c128e --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_placement.py @@ -0,0 +1,197 @@ +"""Placement and rollout-cost contracts for the distributed worker pool. + +The independent review found that mediator-N pinned itself hard to worker-N while +sharing a ReadWriteOnce claim with it, so a drain or preemption that moved only +the worker deadlocked the ordinal on Multi-Attach until an operator deleted a +Pod. It also found that the provider-CLI install was redone on every Pod start +inside the 10m Flux health window for the whole ``hermes`` app. +""" + +from __future__ import annotations + +import json + +import yaml + +from testing.tests.test_hermes_execution_pool_manifests import ( # noqa: E402 + FLUX, + HERMES, + documents, + mediators, + node, + placeable, + worker, +) + + +SCRIPTS = HERMES / "scripts" + + +def test_the_shared_workspace_claim_cannot_deadlock_on_multi_attach(): + claims = { + item["metadata"]["name"]: item["spec"]["accessModes"] + for item in worker()["spec"]["volumeClaimTemplates"] + } + assert claims["workspace"] == ["ReadWriteMany"] + assert claims["tools"] == ["ReadWriteOnce"] + assert claims["provider-access"] == ["ReadWriteOnce"] + + # Every claim the mediator co-mounts with the worker must be multi-attachable. + for ordinal, deployment in enumerate(mediators()): + shared = { + volume["persistentVolumeClaim"]["claimName"] + for volume in deployment["spec"]["template"]["spec"]["volumes"] + if "persistentVolumeClaim" in volume + } + assert f"workspace-hermes-execution-worker-{ordinal}" in shared + for name in shared: + if name.startswith("workspace-hermes-execution-worker-"): + assert claims["workspace"] == ["ReadWriteMany"] + else: + assert name == f"hermes-execution-mediator-state-{ordinal}" + + +def test_mediator_placement_is_a_preference_and_shares_the_worker_priority(): + stateful = worker()["spec"]["template"]["spec"] + worker_terms = stateful["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] + for ordinal, deployment in enumerate(mediators()): + pod = deployment["spec"]["template"]["spec"] + assert pod["priorityClassName"] == stateful["priorityClassName"] == "scavenger" + assert deployment["spec"]["strategy"]["type"] == "Recreate" + affinity = pod["affinity"] + # No hard podAffinity: the mediator must never be unschedulable purely + # because its worker is Pending or placed on a different node. + assert "requiredDuringSchedulingIgnoredDuringExecution" not in affinity["podAffinity"] + preferred = affinity["podAffinity"][ + "preferredDuringSchedulingIgnoredDuringExecution" + ] + assert preferred[0]["podAffinityTerm"]["labelSelector"]["matchLabels"] == { + "statefulset.kubernetes.io/pod-name": f"hermes-execution-worker-{ordinal}" + } + assert affinity["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] == worker_terms + + +def test_a_preemption_that_moves_the_worker_leaves_both_pods_schedulable(): + """The exact drain/pressure sequence that used to need a manual Pod delete.""" + nodes = [node(f"titan-{index:02d}") for index in (5, 6, 7, 8)] + stateful = worker()["spec"]["template"]["spec"] + mediator = mediators()[1]["spec"]["template"]["spec"] + + # Steady state: worker-1 and mediator-1 are colocated on titan-05. + assert "titan-05" in placeable(stateful, nodes) + assert "titan-05" in placeable(mediator, nodes) + + # titan-05 comes under pressure and evicts the scavenger-priority worker, + # which reschedules elsewhere while the mediator has not moved yet. + survivors = placeable(stateful, nodes, occupied=("titan-05",)) + assert survivors and "titan-05" not in survivors + + # The mediator still serves from titan-05 -- the shared claim is RWX, so the + # rescheduled worker does not block on Multi-Attach. + assert "titan-05" in placeable(mediator, nodes) + + # And when titan-05 itself is drained the mediator follows independently, + # because nothing pins it to a Pod that may still be Pending. + drained = [ + node(item["name"], cordoned=item["name"] == "titan-05") for item in nodes + ] + rescheduled = placeable(mediator, drained) + assert rescheduled and "titan-05" not in rescheduled + + +def test_three_workers_still_spread_across_distinct_eligible_nodes(): + stateful = worker()["spec"]["template"]["spec"] + nodes = [node(f"titan-{index:02d}") for index in (5, 6, 7)] + [ + node("titan-13"), node("titan-22"), + ] + occupied: list[str] = [] + for _replica in range(3): + allowed = placeable(stateful, nodes, occupied=tuple(occupied)) + assert allowed, f"replica {len(occupied)} is unschedulable" + occupied.append(allowed[0]) + assert sorted(occupied) == ["titan-05", "titan-06", "titan-07"] + + +def test_provider_clis_install_once_onto_a_durable_verified_volume(): + stateful = worker() + pod = stateful["spec"]["template"]["spec"] + claims = {item["metadata"]["name"] for item in stateful["spec"]["volumeClaimTemplates"]} + assert "tools" in claims + ephemeral = { + volume["name"] for volume in pod["volumes"] if "emptyDir" in volume + } + assert "tools" not in ephemeral, "the CLI cache must survive a Pod restart" + + install = next( + item for item in pod["initContainers"] if item["name"] == "install-provider-clis" + ) + script = install["args"][0] + # The marker alone is not trusted: a pruned or partial cache reinstalls. + assert 'if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ]' in script + assert 'rm -f "${marker}"' in script + # And the install itself is bounded, so a stalled registry fails fast. + assert "timeout 900 npm install" in script + assert "--fetch-timeout=120000" in script + assert script.count("npm install") == 1 + + worker_mount = next( + item for item in pod["containers"][0]["volumeMounts"] + if item["name"] == "tools" + ) + assert worker_mount["readOnly"] is True + + +def test_flux_health_for_hermes_does_not_wait_on_the_best_effort_pool(): + kustomization = documents(FLUX / "hermes/kustomization.yaml")[0] + gated = { + (item["kind"], item["name"]) + for item in kustomization["spec"]["healthChecks"] + } + assert ("StatefulSet", "hermes-execution-worker") not in gated + assert not any(name.startswith("hermes-execution-") for _kind, name in gated) + # The rest of the app's health contract is untouched, so its dependents keep + # the same guarantees they had before the pool existed. + assert ("Deployment", "hermes-agent") in gated + assert ("Deployment", "hermes-switchyard") in gated + assert kustomization["spec"]["timeout"] == "10m" + + for name in ("hermes-chat", "hermes-observer-bindings"): + dependent = documents(FLUX / name / "kustomization.yaml")[0] + assert {item["name"] for item in dependent["spec"]["dependsOn"]} >= {"hermes"} + + +def test_the_pool_still_reports_its_own_readiness(): + pod = worker()["spec"]["template"]["spec"] + assert pod["containers"][0]["readinessProbe"]["exec"]["command"][-1].startswith( + "test -w /workspace" + ) + for deployment in mediators(): + container = deployment["spec"]["template"]["spec"]["containers"][0] + assert container["readinessProbe"]["httpGet"] == { + "path": "/ready", "port": "mediator" + } + assert container["startupProbe"]["failureThreshold"] == 60 + + +def test_the_rendered_pool_configmap_carries_every_mounted_module(): + kustomization = yaml.safe_load((HERMES / "kustomization.yaml").read_text()) + generator = next( + item for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-execution-pool" + ) + keys = {entry.split("=", 1)[0] for entry in generator["files"]} + assert { + "execution_pool_store.py", + "execution_pool_maintenance.py", + "execution_pool_coordinator.py", + "execution_pool_server.py", + } <= keys + modules = sorted( + path.name for path in SCRIPTS.glob("execution_pool_*.py") + ) + assert set(modules) <= keys, "a pool module is missing from its own mount" + assert json.dumps(sorted(keys)) diff --git a/testing/tests/test_hermes_execution_pool_recovery.py b/testing/tests/test_hermes_execution_pool_recovery.py new file mode 100644 index 00000000..e78dd2da --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_recovery.py @@ -0,0 +1,175 @@ +"""Lease-failure recovery contracts for the execution-pool coordinator. + +The independent review reproduced a wedge in which a Kanban write that failed +during lease expiry left a ``lease_failed`` row that was invisible, immortal, +and fatal. These tests pin the recovered behaviour: the row stays retryable, is +only collected on authoritative terminal evidence, and survives a restart on the +PVC the store lives on. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_maintenance as maintenance # noqa: E402 +import execution_pool_store as pool_store # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + MASTER, + assignment_payload, + binding, + install_kanban, + task, +) +from testing.tests.test_hermes_execution_pool_support import ( # noqa: E402 + LOCKED, + age, + clear_pool_deferrals, # noqa: F401 - autouse fixture, resolved by name + exhausted_store, + states, +) + + +def test_locked_board_at_lease_expiry_stays_retryable_and_never_wedges( + tmp_path, monkeypatch, capsys +): + """The exact review repro: a locked board must not strand the run.""" + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + + def locked(*_args, **_kwargs): + raise LOCKED + + kanban.block_task = locked + store = exhausted_store(tmp_path / "wedge.db") + pool = coordinator.Coordinator(MASTER, store) + + pool.expire_leases() + assert states(store) == [("23", 0, 3, "lease_failed")] + assert store.available_ordinals() == [0, 1, 2] + assert "lease park" in capsys.readouterr().err + + # 1. The row is never collected while its outcome is unconfirmed. + age(store, 10 * 365 * 86400) + assert store.garbage_collect(0) == 0 + assert states(store) == [("23", 0, 3, "lease_failed")] + + # 2. reconcile() no longer hits the conflicting-duplicate primary key. + for _cycle in range(3): + pool.reconcile() + assert states(store) == [("23", 0, 3, "lease_failed")] + + # 3. dispatch() makes no spurious Kanban mutation. + monkeypatch.setattr( + maintenance.cli_lane_dispatch, "claim_ready", + lambda *_args, **_kwargs: [("metis", live.id)], + ) + pool.dispatch() + assert kanban.blocked == [] + assert states(store) == [("23", 0, 3, "lease_failed")] + + # 4. Once the board recovers, the retry records the exact run and settles. + kanban.block_task = lambda _connection, task_id, **values: ( + kanban.blocked.append((task_id, values)) or True + ) + pool.expire_leases() + assert kanban.blocked[-1][1] == { + "reason": "Distributed worker lease expired after 3 fenced attempts", + "kind": "transient", + "expected_run_id": 23, + } + assert states(store) == [("23", 0, 3, "finalized")] + + # 5. Only now, with authoritative evidence, is the row collectable. + age(store, 10 * 365 * 86400) + assert store.garbage_collect(0) == 1 + assert states(store) == [] + + +def test_refused_exact_run_park_keeps_the_row_for_the_next_pass(tmp_path, monkeypatch): + """No terminal state without evidence, even when the write is refused.""" + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + kanban.block_task = lambda _connection, task_id, **values: ( + kanban.blocked.append((task_id, values)) or False + ) + store = exhausted_store(tmp_path / "refused.db") + pool = coordinator.Coordinator(MASTER, store) + + pool.expire_leases() + assert states(store) == [("23", 0, 3, "lease_failed")] + assert len(kanban.blocked) == 1 + + # The next pass retries rather than giving up on the run. + pool.expire_leases() + assert len(kanban.blocked) == 2 + assert states(store) == [("23", 0, 3, "lease_failed")] + + +def test_lease_failure_settles_stale_when_the_run_moved_or_vanished( + tmp_path, monkeypatch +): + moved = task(current_run_id=99) + kanban = install_kanban(monkeypatch, [moved], tmp_path) + store = exhausted_store(tmp_path / "moved.db") + coordinator.Coordinator(MASTER, store).expire_leases() + assert states(store) == [("23", 0, 3, "stale")] + assert kanban.blocked == [] + + kanban.tasks.clear() + gone = exhausted_store(tmp_path / "gone.db", binding(task_id="t_gone", attempt=3)) + coordinator.Coordinator(MASTER, gone).expire_leases() + assert states(gone) == [("23", 0, 3, "stale")] + + +def test_lease_failure_settles_finalized_when_kanban_already_parked_the_run( + tmp_path, monkeypatch +): + parked = task(status="blocked") + kanban = install_kanban(monkeypatch, [parked], tmp_path) + store = exhausted_store(tmp_path / "parked.db") + coordinator.Coordinator(MASTER, store).expire_leases() + assert states(store) == [("23", 0, 3, "finalized")] + assert kanban.blocked == [] + + +def test_noncanonical_run_lease_failure_is_released_without_a_board_call( + tmp_path, monkeypatch +): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = exhausted_store( + tmp_path / "bad-run.db", binding(run_id="run-bad", attempt=3) + ) + coordinator.Coordinator(MASTER, store).expire_leases() + assert states(store) == [("run-bad", 0, 3, "stale")] + assert kanban.blocked == [] + + +def test_a_durable_row_in_any_state_blocks_conflicting_re_adoption(tmp_path, monkeypatch): + """known_runs() covers terminal and retrying rows, not just live ones.""" + live = task() + install_kanban(monkeypatch, [live], tmp_path) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(context="different"), + ) + store = pool_store.PoolStore(tmp_path / "known.db") + store.add(binding(), assignment_payload()) + pool = coordinator.Coordinator(MASTER, store) + for state in ("lease_failed", "finalized", "stale"): + with store._connect() as connection: + connection.execute("UPDATE assignments SET state=?", (state,)) + assert store.known_runs() == {("metis", "t_deadbeef", "23")} + pool.reconcile() + assert states(store) == [("23", 0, 1, state)] diff --git a/testing/tests/test_hermes_execution_pool_submission.py b/testing/tests/test_hermes_execution_pool_submission.py new file mode 100644 index 00000000..2ea26692 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_submission.py @@ -0,0 +1,253 @@ +"""Retry-submission contracts against the creation-only Atlas SCM broker. + +The broker deliberately permits only new namespaced branch creation, so a pool +retry that adds commits to an already-published branch could never submit: the +push was refused, the worker unwound, and the run's actual work was discarded +along with it. Submission now targets a fresh attempt- or content-scoped ref, and +a refused submission downgrades the result instead of losing it. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_client as client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +import scm_broker # noqa: E402 +from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402 + KEY, + RESULT, + assignment, + binding, + payload, +) + + +def attempt_assignment(attempt): + """One signed assignment envelope bound to an exact retry attempt.""" + return protocol.sign_envelope( + KEY, "assignment", binding(attempt=attempt), payload() + ) + + +def test_submission_refs_are_creation_only_and_attempt_scoped(): + first = scm.submission_refs("feature/pool", 1, "a" * 40) + assert first[0] == "feature/pool" + assert first[1] == "feature/pool-attempt-1" + assert first[2] == f"feature/pool-{'a' * 12}" + retry = scm.submission_refs("feature/pool", 3, "b" * 40) + assert retry[1] == "feature/pool-attempt-3" + # Every candidate stays inside a reviewed namespace the broker accepts, and + # none of them is a protected or base ref. + for candidate in first + retry: + assert scm_broker.FEATURE_REF_RE.fullmatch(f"refs/heads/{candidate}") + assert candidate not in {"main", "master"} + + +def test_submission_refs_drop_oversized_names_and_fail_closed_when_empty(): + long_branch = "feature/" + "a" * 185 + candidates = scm.submission_refs(long_branch, 2, "c" * 40) + assert candidates == (long_branch,) + with pytest.raises(protocol.ProtocolError, match="reviewed branch name"): + scm.submission_refs("main", 1, "d" * 40) + + +def test_the_broker_accepts_the_attempt_ref_as_a_creation_and_still_refuses_updates(): + zero, head = b"0" * 40, b"1" * 40 + ref = b"refs/heads/feature/pool-attempt-2" + create = b" ".join((zero, head, ref)) + scm_broker._validate_receive_pack( + b"%04x%s\n0000" % (len(create) + 5, create), "token" + ) + update = b" ".join((b"2" * 40, head, ref)) + with pytest.raises(scm_broker.PolicyError, match="new namespaced branch"): + scm_broker._validate_receive_pack( + b"%04x%s\n0000" % (len(update) + 5, update), "token" + ) + + +def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1): + """A Boundary whose broker advertises exactly ``remote_refs``.""" + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + exact = attempt_assignment(attempt) + destination = scm.workspace_path(exact) + destination.mkdir(parents=True) + protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40}) + head = "b" * 40 + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: head) + calls: list[tuple[str, ...]] = [] + + def run(*arguments, **_kwargs): + calls.append(arguments) + if arguments[0] == "ls-remote": + return "\n".join( + f"{sha}\trefs/heads/{ref}" for ref, sha in remote_refs.items() + ) + if arguments[0] == "rev-list": + return "2" + return "" + + monkeypatch.setattr(scm, "_run", run) + boundary = scm.Boundary(KEY) + monkeypatch.setattr(boundary, "_draft", lambda *args: f"https://scm/pulls/{args[1]}") + return boundary, exact, calls, head + + +def test_a_retry_publishes_an_attempt_ref_instead_of_updating_the_existing_one( + tmp_path, monkeypatch +): + boundary, exact, calls, _head = submit_harness( + tmp_path, monkeypatch, {"wt/t_deadbeef": "c" * 40}, attempt=2 + ) + result = boundary.submit(exact, {"title": "retry", "body": "evidence"}) + pushes = [item for item in calls if item[0] == "push"] + assert pushes == [("push", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef-attempt-2")] + assert result["branch"] == "wt/t_deadbeef-attempt-2" + assert result["pull_request"] == "https://scm/pulls/wt/t_deadbeef-attempt-2" + + +def test_an_already_published_head_is_adopted_without_a_second_push( + tmp_path, monkeypatch +): + boundary, exact, calls, head = submit_harness( + tmp_path, monkeypatch, {"wt/t_deadbeef-attempt-2": "b" * 40}, attempt=2 + ) + assert head == "b" * 40 + result = boundary.submit(exact, {"title": "replay", "body": "evidence"}) + assert [item for item in calls if item[0] == "push"] == [] + assert result["branch"] == "wt/t_deadbeef-attempt-2" + assert result["pull_request"].endswith("wt/t_deadbeef-attempt-2") + + +def test_a_run_with_no_new_commits_still_reports_work_a_prior_attempt_published( + tmp_path, monkeypatch +): + boundary, exact, calls, _head = submit_harness( + tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40} + ) + + real_run = scm._run + + def run(*arguments, **kwargs): + if arguments[0] == "rev-list": + return "0" + return real_run(*arguments, **kwargs) + + monkeypatch.setattr(scm, "_run", run) + result = boundary.submit(exact, {"title": "nothing new", "body": "evidence"}) + assert [item for item in calls if item[0] == "push"] == [] + assert result["pull_request"].endswith("wt/t_deadbeef") + + +def test_a_run_that_produced_nothing_at_all_submits_nothing(tmp_path, monkeypatch): + boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, {}) + + real_run = scm._run + + def run(*arguments, **kwargs): + if arguments[0] == "rev-list": + return "0" + return real_run(*arguments, **kwargs) + + monkeypatch.setattr(scm, "_run", run) + result = boundary.submit(exact, {}) + assert [item for item in calls if item[0] == "push"] == [] + assert result["pull_request"] == "" and result["branch"] == "wt/t_deadbeef" + + +def test_submission_fails_closed_when_every_candidate_ref_is_taken( + tmp_path, monkeypatch +): + taken = { + "wt/t_deadbeef": "c" * 40, + "wt/t_deadbeef-attempt-1": "d" * 40, + f"wt/t_deadbeef-{'b' * 12}": "e" * 40, + } + boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, taken) + with pytest.raises(protocol.ProtocolError, match="every reviewed branch name"): + boundary.submit(exact, {"title": "blocked", "body": "evidence"}) + assert [item for item in calls if item[0] == "push"] == [] + + +def test_remote_head_parsing_ignores_anything_that_is_not_a_branch( + tmp_path, monkeypatch +): + monkeypatch.setattr( + scm, "_run", + lambda *_a, **_k: "aaa\trefs/tags/v1\nbbb\trefs/heads/feature/x\ngarbage\n", + ) + assert scm._remote_heads(tmp_path, ("feature/x",)) == {"feature/x": "bbb"} + + +class FailingSCM: + def __init__(self, error): + self.error = error + self.calls = 0 + + def submit(self, _assignment, _request): + self.calls += 1 + raise self.error + + +def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it(): + failing = FailingSCM(protocol.ProtocolError("every reviewed branch name is taken")) + boundary = client.ClientBoundary(KEY, failing) + boundary.current = assignment() + boundary._post = lambda _path, _envelope: protocol.sign_envelope( + KEY, "ack", binding(), {"accepted": True, "duplicate": False} + ) + finished = boundary.finish( + { + "binding": binding(), + "payload": {"structured": dict(RESULT), "returncode": 0}, + "title": "safe", "body": "evidence", + } + ) + assert failing.calls == 1 + structured = finished["structured"] + # The run is recorded rather than unwound, and it says exactly why. + assert structured["status"] == "blocked" + assert structured["changed_files"] == ["safe.py"] + assert len(structured["blockers"]) == 1 + assert "commits remain on this ordinal's workspace" in structured["blockers"][0] + assert "every reviewed branch name is taken" in structured["blockers"][0] + assert finished["ack"]["accepted"] is True + + +def test_a_successful_submission_records_both_the_draft_and_the_exact_branch(): + class Recording: + def submit(self, _assignment, _request): + return { + "workspace": "/workspace/runs/metis/t_deadbeef/42", + "branch": "wt/t_deadbeef-attempt-2", + "pull_request": "https://scm/pulls/9", + } + + boundary = client.ClientBoundary(KEY, Recording()) + boundary.current = assignment() + boundary._post = lambda _path, _envelope: protocol.sign_envelope( + KEY, "ack", binding(), {"accepted": True, "duplicate": False} + ) + finished = boundary.finish( + { + "binding": binding(), + "payload": {"structured": dict(RESULT), "returncode": 0}, + "title": "safe", "body": "evidence", + } + ) + assert finished["structured"]["status"] == "completed" + assert finished["structured"]["artifacts"] == [ + "https://scm/pulls/9", "branch:wt/t_deadbeef-attempt-2", + ] diff --git a/testing/tests/test_hermes_execution_pool_support.py b/testing/tests/test_hermes_execution_pool_support.py new file mode 100644 index 00000000..d98fd373 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_support.py @@ -0,0 +1,60 @@ +"""Shared fixtures for the execution-pool recovery and isolation suites.""" + +from __future__ import annotations + +import sqlite3 +import sys +import time +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_maintenance as maintenance # noqa: E402 +import execution_pool_store as pool_store # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + assignment_payload, + binding, +) + + +LOCKED = sqlite3.OperationalError("database is locked") + + +@pytest.fixture(autouse=True) +def clear_pool_deferrals(): + """Keep the bounded deferral report from leaking between tests.""" + maintenance.DEFERRALS.clear() + yield + maintenance.DEFERRALS.clear() + + +def exhausted_store(path, exact=None): + """A store whose only assignment has burned every fenced attempt.""" + store = pool_store.PoolStore(path) + store.add(exact or binding(attempt=3), assignment_payload()) + store.offer(int((exact or binding(attempt=3))["worker_ordinal"])) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + return store + + +def states(store): + """Every durable row as (run_id, ordinal, attempt, state).""" + return [ + tuple(row) + for row in store._connect().execute( + "SELECT run_id,worker_ordinal,attempt,state FROM assignments ORDER BY run_id" + ) + ] + + +def age(store, seconds): + """Backdate every row so retention-based collection can be exercised.""" + with store._connect() as connection: + connection.execute("UPDATE assignments SET updated_at=?", (time.time() - seconds,)) From 54156f839efd7de3864bda9d0f625396b8fb2d36 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:47:24 +0000 Subject: [PATCH 22/46] docs(hermes): record the repaired pool operational boundaries Document what an operator needs to know about the repairs: why the ordinal workspace claim is ReadWriteMany and why colocation is only a preference, that `lease_failed` is retryable rather than terminal and what a persistent deferred park means, why a retry publishes an attempt-scoped ref and what to expect from the extra drafts, and why the best-effort pool is deliberately absent from the hermes Kustomization health checks. Co-Authored-By: Claude Opus 5 --- services/hermes/NOTES.md | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index 400bf44f..bd8437f5 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -275,9 +275,37 @@ Use this short explanation: one RWO checkout. Repository and base-branch identity comes from the canonical board registry rather than task-supplied metadata. - Model Pods have no pool key, broker mount, broker egress, or Kubernetes token. - Ordinal mediator Deployments are separate network identities colocated with - the matching workspace PVC; they alone authenticate exact-run results and - reach the PR14 SCM broker. + Ordinal mediator Deployments are separate network identities that co-mount the + matching workspace PVC; they alone authenticate exact-run results and reach the + PR14 SCM broker. +- The ordinal workspace claim is `ReadWriteMany` because the worker and its + mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved + only one of them left the other holding the attachment on the old node, and + the ordinal deadlocked on Multi-Attach until an operator deleted a Pod. + Colocation is now a scheduling preference and both Pods run at `scavenger` + priority, so either can be evicted and rescheduled on its own. There is + deliberately no PodDisruptionBudget: this is a best-effort pool and a budget + would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`. +- `lease_failed` is a *retryable* durable state, not a terminal one. It means the + ordinal was released but Kanban has not confirmed the outcome, so every + maintenance pass retries it. Only `finalized` and `stale` are terminal, and + only terminal rows are garbage-collected, so no run is dropped before the pool + knows what happened to it. If the log keeps reporting a deferred lease park for + the same run, Kanban is refusing an exact-run park on a task it still reports + as running: inspect that task rather than deleting the row. +- The broker accepts branch creation only. A pool retry that adds commits + therefore publishes a new `-attempt-` (or, failing that, + `-`) ref in the same reviewed namespace instead of updating the + ref a previous attempt published, and adopts an existing ref that already + points at the exact head rather than pushing again. Expect one draft per + published ref; each is still human-reviewed. A submission that cannot land + blocks the run with the reason and leaves the commits on the ordinal's + workspace, so prior work is recoverable. +- Provider CLIs install once per pinned version onto a durable per-ordinal + `tools` claim, re-verified against the real binaries so a pruned cache + reinstalls. The pool is intentionally absent from the `hermes` Kustomization's + `healthChecks`: gating that 10m window on a best-effort pool would stall + `hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`. - A hashed execution-pool ConfigMap, protocol-version readiness checks, and versioned SCM boundary name make code/config changes controlled rollouts. A rollout is still a human-reviewed operation; this repository change does not From 360d8aad8f6033f6088423b4dfd5d400938c2144 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 17:22:43 +0000 Subject: [PATCH 23/46] hermes: close the review-role gaps found reviewing PR 22 Repairs the blockers from the independent review of the previous head. The role-aware verdict contract was correct but reachable only through one call site and only on cards written with real newlines, so most real review cards never used it. * The role-blind call is no longer a weaker classifier that can reject before the role-aware judge runs. Without a card there is no defensible role-dependent judgement, so `unfinished_result_reason()` applies only the card-independent checks. That closes the short-circuit at every call site, including the one PR15 moves to `cli_lane_execution`, and it makes a journalled terminal record accepted under one version of these semantics re-validate under any other instead of being quarantined into a re-dispatch of an already-accepted task. * The lane resolves the role from the card and passes it, and records it in the Kanban metadata. The verdict contract binds only where the lane can buy another turn: in single-shot mode a rejection discards the worker's real result, so review cards keep the relaxation without gaining any rejection single-shot mode did not already have. * Card scope expands the literal \n escapes the board stores in one-line bodies, so explicit `Hermes-Task-Role` / `Hermes-Expected-Output` directives are honoured on the 6 of 78 live cards that carry no real newline, and the read-only, verdict and mutation heuristics stop being cut apart by them. * Card scope now ends at the first non-card H2 and at the runner's controller evidence, which is emitted under its own heading. Goal-controller rejection history can no longer sit inside the card, and an upstream heading rename fails closed instead of admitting history into role resolution. * The inference recognises the SHIP/BLOCK-shaped deliverables real cards actually use: 21 of 78 live cards resolve to review, up from 10, with no implementation card misclassified. Cards asking for a findings list rather than a verdict deliberately stay on the model judge, since the verdict is the review contract's only gate. * A declared review role no longer outranks mutation evidence: a report that changed files falls back to the implementation regime. * Judge reasons go through the agent runtime's canonical redactor, extended for the two shapes it deliberately passes through and this lane handles - `scheme://user:secret@host` and a credential named in prose - while a 40-hex commit SHA survives as evidence. Regressions cover the recovered t_dbdcd739 incident, a verbatim snapshot of every live card on three boards with a hand-labelled expected role, the upgrade and single-shot properties against the previous gate, the upstream context-heading contract, and the end-to-end `execute_claim` shape that used to burn every goal turn. Co-Authored-By: Claude Opus 5 --- services/hermes/scripts/cli_lane_goal.py | 210 ++++++++---- services/hermes/scripts/cli_lane_runner.py | 17 +- .../data/hermes_kanban_card_corpus.jsonl | 78 +++++ .../tests/data/hermes_t_dbdcd739_context.txt | 73 ++++ .../tests/data/hermes_t_dbdcd739_result.json | 47 +++ testing/tests/test_hermes_cli_lane_goal.py | 6 +- .../tests/test_hermes_cli_review_contract.py | 313 ++++++++++++++++++ .../tests/test_hermes_cli_review_corpus.py | 212 ++++++++++++ testing/tests/test_hermes_cli_review_goal.py | 31 +- testing/tests/test_hermes_cli_review_roles.py | 159 +++++++++ 10 files changed, 1076 insertions(+), 70 deletions(-) create mode 100644 testing/tests/data/hermes_kanban_card_corpus.jsonl create mode 100644 testing/tests/data/hermes_t_dbdcd739_context.txt create mode 100644 testing/tests/data/hermes_t_dbdcd739_result.json create mode 100644 testing/tests/test_hermes_cli_review_contract.py create mode 100644 testing/tests/test_hermes_cli_review_corpus.py create mode 100644 testing/tests/test_hermes_cli_review_roles.py diff --git a/services/hermes/scripts/cli_lane_goal.py b/services/hermes/scripts/cli_lane_goal.py index 0cf991f5..2c6f78c4 100644 --- a/services/hermes/scripts/cli_lane_goal.py +++ b/services/hermes/scripts/cli_lane_goal.py @@ -12,12 +12,37 @@ health of whatever the card asked the worker to look at. the deliverable, not a task blocker, so the card is never resumed with an instruction to repair an implementation the reviewer may not touch. +Three deliberate boundaries keep that regime honest. First, ``role`` belongs to +the caller that holds the card: ``unfinished_result_reason`` cannot classify a +card it was never handed, so every caller holding the objective MUST pass +``role``; the role-blind default is the replay contract below, never a weaker +classifier that could short-circuit the role-aware gate. + +Second, the verdict contract binds in single-shot mode exactly as in the goal +loop: a review card's deliverable *is* its verdict, so the lane fails closed +rather than record a verdictless review as done. Two single-shot outcomes change +against main d8f2d818, deliberately - a completed review whose prose calls the +*reviewed* artifact pending now completes instead of being discarded, and one +carrying no usable verdict now blocks instead of completing. + +Third, a journalled terminal record stays valid across an upgrade. PR15 re-runs +this gate on replay without a card, so ``role=None`` keeps the unfinished-work +heuristic - that integrity check still works - and skips it only for a report +that satisfies the whole review contract and changed no files. That is precisely +the set the role-aware lane accepts as a review, so a record accepted under one +version re-validates under every other instead of being quarantined into a +re-dispatch of an already-accepted task. + The task role comes from explicit card metadata (``Hermes-Task-Role: review``) -whenever the card author supplies it. Cards written before that contract fall -back to a deliberately narrow inference that requires a read-only scope, a -requested verdict, no requested mutation deliverable, and a report that changed -no files. Review completion is fail-closed on a missing, unrecognized, or -self-contradictory verdict and on missing evidence. +whenever the card author supplies it, read from real newlines and from the +literal ``\\n`` escapes the board stores in single-line bodies. Pre-contract +cards fall back to a narrow inference needing a read-only scope, a requested +SHIP/BLOCK verdict, no requested mutation deliverable, and a report that changed +no files. A card whose deliverable is a findings list rather than a verdict +deliberately stays on the implementation regime: the verdict *is* the review +contract's gate, so inferring a review role for a card that never asked for one +could only fail closed. A report that changed files never resolves to the review +role, even when the card declares it. """ from __future__ import annotations @@ -29,6 +54,11 @@ import urllib.error import urllib.request from typing import Any, Callable +try: # The lane runs inside the agent image, which owns the canonical redactor. + from agent.redact import redact_sensitive_text as _canonical_redact +except Exception: # pragma: no cover - only when the agent runtime is absent + _canonical_redact = None + GOAL_JUDGE_URL = os.environ.get( "HERMES_GOAL_JUDGE_URL", @@ -67,6 +97,22 @@ READ_ONLY_GUARD = ( "the assigned action is a read-only review: report the verdict and evidence, " "and do not modify the reviewed implementation" ) +# The runner appends its own turn/rejection evidence to the judge objective. +# It is emitted under this heading so ``card_scope`` drops it with every other +# non-card section instead of letting a quoted worker sentence reassign a role. +CONTROLLER_EVIDENCE_HEADING = "## Hermes goal-controller evidence" +# ``hermes_cli.kanban_db.build_worker_context`` (Hermes runtime 0.18.2) renders +# the worker context as H2 sections and emits exactly CARD_SECTIONS followed by +# HISTORY_SECTIONS; ``test_hermes_cli_review_context.py`` pins that contract +# against the installed runtime. The cut below allow-lists the two card +# sections, so an upstream rename fails closed - the card just ends early. +CARD_SECTIONS = ("Body", "Attachments") +HISTORY_SECTIONS = ( + "Prior attempts on this task", + "Parent task results", + "Recent work by", + "Comment thread", +) GOAL_JUDGE_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, @@ -86,12 +132,9 @@ UNFINISHED_EVIDENCE = re.compile( r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)", re.IGNORECASE | re.DOTALL, ) -# The worker context appends prior attempts, parent results, cross-task history -# and comments after the card the worker was actually given. Role resolution -# reads only the card itself so an unrelated sentence quoted from an earlier run -# cannot reassign the task role. HISTORY_SECTION = re.compile( - r"^##\s+(?:Prior attempts\b|Parent task results\b|Recent work by\b|Comment thread\b)", + r"^##\s+(?!(?:Body|Attachments)\b)" + r"|^Authoritative Hermes goal-controller evidence\b", re.IGNORECASE | re.MULTILINE, ) # Explicit, machine-readable card metadata. This is the supported contract; @@ -106,24 +149,28 @@ OUTPUT_DIRECTIVE = re.compile( ) READ_ONLY_SCOPE = re.compile( r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?" - r"(?:edit|modify|change|alter|patch|rewrite|implement|repair)\b" + r"(?:edit|modify|change|alter|patch|rewrite|implement|repair|author)\b" r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b" r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b" - r"|\bno\s+(?:code|file|source|implementation)\s+(?:edits?|changes?|modifications?)\b" + r"|\bno\s+(?:\w+\s+){0,2}?(?:edits?|changes?|modifications?)\b" r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b" - r"|\bread[\s-]?only\s+(?:scope|review|audit|analysis|assessment|inspection|diagnos\w+)\b)", + r"|\bpristine\b|\bread[\s-]?only\b)", re.IGNORECASE, ) +# A SHIP/BLOCK-shaped deliverable, not merely a mention of the word "verdict": +# cards that audit some other component's verdict machinery must not be read as +# owing one themselves. VERDICT_DELIVERABLE = re.compile( - r"(?:\bship\s+or\s+block\b|\bblock\s+or\s+ship\b|\bship\s*/\s*block\b" - r"|\breturn\s+(?:a\s+|one\s+|the\s+)?(?:strict\s+)?verdict\b" - r"|\b(?:review|audit|release|diagnostic)\s+verdict\b|\bverdict\s*[:=])", + r"(?:\bship\b[^.\n]{0,120}?\bblock\b|\bblock\b[^.\n]{0,120}?\bship\b" + r"|\bship\s*/\s*not-?ship\b|\bverdict\s+of\b" + r"|\b(?:return|record|report|give|provide|deliver|state|produce|emit)\b" + r"(?:\s+[-\w']+){0,6}?\s+verdict\b)", re.IGNORECASE, ) MUTATION_DELIVERABLE = re.compile( r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b" - r"|\bpush\s+(?:the\s+|your\s+|one\s+)?(?:branch|commit|change\w*|fix\w*)\b" - r"|\bcommit\s+and\s+push\b" + r"|\bpush\s+(?:\w+\s+){0,3}?(?:branch|commit|change\w*|fix\w*)\b" + r"|\bcommit\s+(?:\w+\s+){0,4}?and\s+push\b|\bamend\b" r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)" r"|\bdraft\s+pr\s+only\b)", re.IGNORECASE, @@ -135,11 +182,23 @@ VERDICT_DIRECTIVE = re.compile( ) VERDICT_TOKEN = re.compile(r"(? str: @@ -152,9 +211,17 @@ def _bounded(value: str, limit: int) -> str: return f"{value[:head]}{marker}{value[-(remaining - head):]}" +def redact(value: str) -> str: + """Redact with the runtime's canonical helper plus this lane's own gaps.""" + text = SECRET_LIKE.sub("[redacted]", value) + text = URL_USERINFO.sub(r"\1:[redacted]@", text) + text = CREDENTIAL_CONTEXT.sub(r"\1[redacted]", text) + return _canonical_redact(text, force=True) if _canonical_redact else text + + def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str: """Return one bounded, single-line, secret-free judge reason.""" - text = SECRET_LIKE.sub("[redacted]", str(value or "")) + text = redact(str(value or "")) text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split()) if not text: return "local judge supplied no reason" @@ -169,8 +236,16 @@ def _strings(value: Any) -> list[str]: def card_scope(objective: str) -> str: - """Return the card the worker was assigned, without appended run history.""" - text = str(objective or "") + """Return the card the worker was assigned, without appended run history. + + Literal ``\\n`` escapes are expanded first: the board stores most bodies on + a single physical line, and every pattern here is line-anchored or depends + on word boundaries the escape sequence would otherwise destroy. Expanding + before the cut is monotone - it can only move the first history heading + earlier - so it cannot smuggle appended history into the card. The + normalization is for analysis only; the judge objective is never rewritten. + """ + text = str(objective or "").replace("\\r\\n", "\n").replace("\\n", "\n") match = HISTORY_SECTION.search(text) return text[: match.start()] if match else text @@ -198,10 +273,14 @@ def _declared_role(objective: str) -> str | None: def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]: """Resolve the assigned task role and how that resolution was reached.""" text = card_scope(objective) + changed = _strings((result or {}).get("changed_files")) declared = _declared_role(text) + if declared == REVIEW_ROLE and changed: + # Mutation evidence outranks a declaration: a card cannot label itself + # read-only and then self-certify a report that edited the tree. + return IMPLEMENTATION_ROLE, "conflict" if declared is not None: return declared, "directive" - changed = _strings((result or {}).get("changed_files")) if ( not changed and READ_ONLY_SCOPE.search(text) @@ -253,15 +332,40 @@ def review_completion_problem(result: dict[str, Any]) -> str | None: return None -def _reported_role(result: dict[str, Any]) -> str: - """Classify a report shape when no objective is available to the caller.""" - if _strings(result.get("changed_files")): - return IMPLEMENTATION_ROLE - if result.get("verdict") is not None or VERDICT_DIRECTIVE.search( - str(result.get("summary") or "") +def _completion_problem(result: dict[str, Any], role: str | None) -> str | None: + """Return the unguarded reason this report is not a finished deliverable.""" + status = str(result.get("status") or "") + if status == "incomplete": + summary = str(result.get("summary") or "").strip() + return summary or "worker explicitly reported incomplete work" + if status != "completed": + return None + blockers = result.get("blockers") + if isinstance(blockers, list) and any(str(item).strip() for item in blockers): + return "worker reported blockers while claiming completion" + if role == REVIEW_ROLE: + # A review's prose describes the reviewed artifact, so the unfinished + # work heuristic below would read the artifact's state as the review's + # own. The verdict contract is the deterministic gate instead. + return review_completion_problem(result) + if ( + role is None + and not _strings(result.get("changed_files")) + and review_completion_problem(result) is None ): - return REVIEW_ROLE - return IMPLEMENTATION_ROLE + # A replay caller holds no card. This is exactly the shape the + # role-aware lane accepts as a review, so honouring it here keeps a + # journalled record valid without weakening the heuristic for anything + # that is not already a complete review deliverable. + return None + tests = result.get("tests_run") + evidence = [str(result.get("summary") or "").strip()] + if isinstance(tests, list): + evidence.extend(str(item) for item in tests) + match = UNFINISHED_EVIDENCE.search("\n".join(evidence)) + if match: + return f"completion evidence says work is unfinished: {match.group(0).strip()}" + return None def unfinished_result_reason( @@ -269,29 +373,19 @@ def unfinished_result_reason( *, role: str | None = None, ) -> str | None: - """Reject self-contradictory completion reports without model inference.""" - status = str(result.get("status") or "") - summary = str(result.get("summary") or "").strip() - blockers = result.get("blockers") - if status == "incomplete": - return sanitize_reason(summary or "worker explicitly reported incomplete work") - if status != "completed": + """Reject self-contradictory completion reports without model inference. + + ``role`` must be supplied by any caller holding the card; see the module + docstring for the role-blind replay contract a caller without one gets. + """ + problem = _completion_problem(result, role) + if problem is None: return None - if isinstance(blockers, list) and any(str(item).strip() for item in blockers): - return "worker reported blockers while claiming completion" - if (role or _reported_role(result)) == REVIEW_ROLE: - # A review's prose describes the reviewed artifact, so the unfinished - # work heuristic below would read the artifact's state as the review's - # own. The verdict contract is the deterministic gate instead. - return review_completion_problem(result) - tests = result.get("tests_run") - evidence = [summary] - if isinstance(tests, list): - evidence.extend(str(item) for item in tests) - match = UNFINISHED_EVIDENCE.search("\n".join(evidence)) - if match: - return f"completion evidence says work is unfinished: {match.group(0).strip()}" - return None + if role == REVIEW_ROLE: + # Every reason a reviewer may ever read carries the guard, so a resumed + # review is never told to repair the implementation it must not touch. + problem = f"{READ_ONLY_GUARD}; {problem}" + return sanitize_reason(problem) def _completion_claimed(result: dict[str, Any]) -> bool: @@ -354,16 +448,16 @@ def judge_goal_completion( ) -> tuple[bool, str]: """Judge one worker report against the action its card actually assigned.""" role, source = task_role(objective, result) - deterministic_reason = unfinished_result_reason(result, role=role) - if deterministic_reason is None and not _completion_claimed(result): - deterministic_reason = ( + reason = unfinished_result_reason(result, role=role) + if reason is None and not _completion_claimed(result): + reason = ( f"worker reported status {str(result.get('status') or 'unknown')!r} " "rather than a finished task" ) - if deterministic_reason: if role == REVIEW_ROLE: - return False, sanitize_reason(f"{READ_ONLY_GUARD}; {deterministic_reason}") - return False, sanitize_reason(deterministic_reason) + reason = f"{READ_ONLY_GUARD}; {reason}" + if reason: + return False, sanitize_reason(reason) if role == REVIEW_ROLE: # The requested deliverable is present and self-consistent. A verdict # that the reviewed artifact must not ship is a finished review, so the diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 3eba3453..ea7633ef 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -1005,7 +1005,20 @@ def execute_claim(board: str, task_id: str) -> None: completion_problem = None if structured and result.returncode == 0: - completion_problem = cli_lane_goal.unfinished_result_reason(structured) + # Resolve the role from the card itself, before any + # controller evidence is appended, and hand it to the gate + # so a review is judged on its verdict rather than on prose + # about the artifact it reviewed. + task_role, role_source = cli_lane_goal.task_role( + context, + structured, + ) + metadata["task_role"] = task_role + metadata["task_role_source"] = role_source + completion_problem = cli_lane_goal.unfinished_result_reason( + structured, + role=task_role, + ) if ( structured.get("status") == "completed" and completion_problem is None @@ -1018,7 +1031,7 @@ def execute_claim(board: str, task_id: str) -> None: judge_context = context if goal_turn > 1 or rejection_history: judge_context += ( - "\n\nAuthoritative Hermes goal-controller evidence: " + f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n" f"current turn {goal_turn}/{goal_max_turns}; prior rejected " f"reports: {json.dumps(rejection_history[-5:])}." ) diff --git a/testing/tests/data/hermes_kanban_card_corpus.jsonl b/testing/tests/data/hermes_kanban_card_corpus.jsonl new file mode 100644 index 00000000..15e0f307 --- /dev/null +++ b/testing/tests/data/hermes_kanban_card_corpus.jsonl @@ -0,0 +1,78 @@ +{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, add focused regression tests using only generic fictional card/deck/commander metadata. Before changing production code, verify each new test fails against the current implementation. Cover: (1) reject unsupported exact-cost wording including \"requires {3}{G}{G} to cast\" and \"need five mana to cast\"; (2) reject exhaustive whole-deck statements such as \"every card in the deck leaves the commander cost unchanged,\" including superficial supplied-metadata hedges that still quantify over the entire deck; (3) prove that deterministic claim repair which corrects a commander name is followed by semantic-compliance validation, so a newly introduced exact-cost claim cannot bypass checks; and (4) permit explicitly polarity-reversed caveats/negations such as \"do not assume no card in the whole deck changes the cost.\" Preserve and extend nearby test style and assertions without weakening fail-closed entry-hazard, unknown-card, legal-target, or provenance coverage. Do not implement production fixes, commit, push, deploy, or provide a final structured task result; leave the RED tests and report their locations and observed failures to the implementation owner.", "expected_role": "implementation", "id": "t_14985b68", "title": "Add RED regressions for the four semantic-retry compliance bypasses"} +{"board": "cassandra", "body": "First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Perform an independent high-tier Claude review of the Codex-authored implementation and live evidence for COLLAB-001, COLLAB-002, and COLLAB-011. Check persistence and integrity of both hash types, determinism and exactness of action-ranking replay, worker_app_version provenance, deployment/version alignment with 0.9.63, and whether recorded evidence actually supports each claim. Do not author implementation; identify findings with severity, exact reproduction or inspection commands, and explicit certify/conditional-certify/reject recommendation. Record review artifacts, model/provider/effort, and capacity blockers.", "expected_role": "implementation", "id": "t_15498b8f", "title": "Independently review v69 certification implementation and evidence"} +{"board": "cassandra", "body": "Repair the SAME open atlas/cassandra PR #1 branch handoff/generated-strategy-audit-20260813 starting from exact independently reviewed head b684e6ab6abebd19cbb4a434a8c4fa0e5b7015d6 and API-confirmed base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Parent t_15f2a75c is authoritative; reproduce every exact finding RED end-to-end through section_pipeline with PYTHONPATH explicitly pinned to the intended checked-out tree before modifying production code. Make the smallest structural repair for all four findings without regressing previously verified behavior: (1) normalize/strip Markdown emphasis consistently BEFORE table numeric/cost scanning so _nine_, __nine__, _9_, _nine mana_ cannot hide false values while ordinary identifiers/words and correct decorated/symbolic values remain correct. (2) Do not use a five-name closed header list to decide whether a cost table is scoped. Recognize realistic card identity headers such as Commander, Cards, Spell Name and variants, and preferably bind rows by exact known-card occurrence when a casting/mana-value/CMC cost column exists; ambiguous cost rows must fail closed, activation-only rows remain occurrence-locally exempt. (3) Analyze exact-cost claim text in every relevant table cell, including the nominal card cell and GFM-absorbed pipe-bearing prose/list rows directly following a table, so the exact reviewer examples block; do not join headings, fences, blank-separated paragraphs or disconnected genuine tables, and do not create false positives for ordinary non-claim cell text. (4) Replace the two-word sixth/tenth special case with a grammar/fail-closed exact-cost cue: any lexical/numeric phrase asserted as mana value/CMC/casting cost that is not a proved supported value must block, including zeroth/fourth/seventh/eighth/eleventh and cardinals twenty/twenty-five/thirty/one hundred, without treating often/attention/none/someone/Stoneforge/one-shot/twofold or cautious/negated prose as an exact claim. Add focused tracked regressions for all parent repros, nearby header/emphasis/GFM/number boundaries, and positive controls. Preserve the prior fixes for four/fourteen, six/sixteen, ten/tenth/often, malformed width, activation scope, snapshot fail-closed, one index per call, bounded runtime, unrelated tracked/untracked files and inherited audit hashes. Run focused compliance, semantic/entry, Program, deterministic acceptance and full dynamic authoring suites. For every head comparison and test invocation, pin PYTHONPATH to that exact worktree and prove the loaded module __file__ belongs to it; repeat exact old-head RED and neutered-gate RED. Run py_compile, discovery, diff/LOC, and Ruff if truly available. Commit and push one focused follow-up to the same existing PR branch, verify local/remote/refs-pull/API exact head equality and open/unmerged/undeployed state. Do not merge, deploy, comment on the PR, or modify unrelated files.", "expected_role": "implementation", "id": "t_15ed4f67", "title": "Repair Cassandra PR 1 after b684e6ab review"} +{"board": "cassandra", "body": "Independently review Cassandra PR #1 at exact head b684e6ab6abebd19cbb4a434a8c4fa0e5b7015d6 against base 4a0ea87da71491d02a5ea65de39fa7c613de886d and the four findings in parent review t_b3a79a92. Use Claude Code subscription OAuth only through the assigned cli-claude-xhigh route; do not use any metered API key. Work read-only in a newly created clean detached worktree at the exact head; never edit the implementation workspace, commit, push, comment, merge, deploy, or mutate Kanban except returning this task result. Do not trust the parent completion result, docs, done labels, claimed counts, or existing tests as truth. Inspect the complete base-to-head diff including every new and untracked implementation/test file in the PR. Reproduce the exact prior false-acceptance and bypass inputs: four versus fourteen, six versus sixteen, ten versus tenth and often, ordinary words/card names containing number substrings, wrapped continuation lines containing ordinary pipes, and under-width/wide/misaligned Markdown rows. Add temporary independent probes outside the repo when needed; do not retain them. Verify correct decorated/prose costs, activation exemptions, genuine tables, headings/blank paragraphs/fences, fail-closed snapshot and exception paths, bounded/cardinality/index reuse/runtime behavior, test discovery and non-vacuity. Run focused compliance, semantic/entry, Program, deterministic acceptance, broad authoring suites, old-head RED and neutered-gate RED where independently meaningful. Verify exact local/remote/pull/API head equality and that PR remains open/unmerged/undeployed without exposing credentials. Return a strict findings-first SHIP or BLOCK result with exact file/line and executable repro evidence for every issue; if no defect is found, state residual risks and the evidence supporting SHIP.", "expected_role": "review", "id": "t_15f2a75c", "title": "Independent release review of Cassandra b684e6ab"} +{"board": "cassandra", "body": "Repair every material finding from parent review t_4a0eb5f2 on private atlas/cassandra PR #1, exact starting head 49411de1df18cb8f8f9f792875175a65e9000392. Work on the existing handoff/generated-strategy-audit-20260813 branch and canonical atlas/cassandra remote.\n\nPreserve the four inherited untracked files byte-for-byte and do not stage or commit them:\n- docs/generated_strategy_job_audit.md 0e5c9160...\n- scripts/audit_generated_strategy_job.py 8259a7d8...\n- tests/test_audit_generated_strategy_job.py dbd2d776...\n- uv.lock f2d7ace7...\n\nImplement general, bounded repairs rather than phrase-specific patches:\n1. Parse valid Markdown tables structurally with optional outer pipes, escaped-pipe awareness, column-order independence, and per-cell occurrence scope. Distinguish activation-cost columns from casting/mana-cost columns even in mixed rows and misleading compound headers. Keep malformed input fail-closed.\n2. Cover common exact casting-cost constructions, including subject-first, passive, CMC/mana-value, colon, and existing forms, without flagging activation-cost claims.\n3. Bound unknown/caveat scope across subordinating and coordinating clause boundaries including although/while/but/semicolon.\n4. Cover common whole-deck closure constructions under incomplete metadata.\n5. Never let a strategy antecedent waive explicitly cost-typed anaphors; accept natural explicit strategy antecedents such as “Our strategy centers on … changes it” while keeping bare or ambiguous anaphors fail-closed.\n6. Correct documentation claims to match actual generalized behavior.\n\nBefore production edits, add tracked fictional regressions that reproduce every HIGH/MEDIUM finding and the cautious controls. Preserve ordinary valid prose/table positives. Keep every changed production/test file below 500 LOC by cohesive domain splits, not part/chunk naming. Avoid named-card production branches.\n\nRun the expanded focused compliance and release-blocker suites, semantic-review suite, Program-quality suite, structure/migration/docs/LOC/static/frontend/diff checks, bounded runtime probes, and a conclusive broad strategy-authoring discovery suite because the grammar surface changes broadly. Clearly distinguish newly run evidence from inherited evidence.\n\nWhen all required gates pass, update the collaboration/devlog evidence accurately, commit only intended tracked files, push the same branch, and verify local HEAD, remote branch, and open PR head match. Do not merge, deploy, certify live guides, or comment in Gitea. If work remains or any required test is still running, return incomplete so the goal loop continues.", "expected_role": "implementation", "id": "t_1e473bd7", "title": "Repair Cassandra PR 1 findings from independent v4 review"} +{"board": "cassandra", "body": "Runtime-only acceptance canary for the external Kanban goal loop. Do not run tools or commands, read or edit files, access the network, or mutate any external state. On the first provider turn, return the required structured result with status=incomplete, summary exactly PHASE_ONE_INCOMPLETE, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary phase one. On the resumed provider turn, return status=completed, summary exactly GOAL_LOOP_OK, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary: passed. Do not return completed on the first turn. Completion is valid only after Hermes rejects the first report and resumes this same card.", "expected_role": "implementation", "id": "t_21496dc5", "title": "Goal-loop completion canary"} +{"board": "cassandra", "body": "Review the implemented semantic-repair correction in /opt/data/workspace/projects/cassandra-hermes-v69 against the regression requirements and run the required validation without changing behavior unless a test infrastructure issue requires a clearly justified minimal fix. Execute the focused semantic-repair/generator tests, the relevant broad generator suite, and the complete 9/9 acceptance check required by project guidance. Confirm that the repair remains generic, bounded, fail-closed for entry hazards and unknown cards, restricted to legal targets, and provenance-backed. Capture exact commands, pass/fail counts, and any skips or environmental limitations for handoff. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_237244f2", "title": "Run semantic-repair generator and acceptance verification"} +{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, read AGENTS.md and docs/agent_collaboration.md before editing. Locate the semantic-repair and generator test harnesses, then add metadata-driven fictional regression coverage that first fails against the current behavior. Cover: (1) retry repair must fail closed when a candidate route retains or introduces an entry hazard, including a hazardous ETB/entry route analogous to the reported Lazav failure; and (2) retry repair must not retain or introduce unsupported exact commander-cost or whole-deck closure claims when required facts/provenance are incomplete, analogous to the Goreclaw failure. Ensure fixtures and assertions are generic and do not name or special-case production cards. Preserve coverage for unknown-card, legal-target, provenance, and fail-closed hazard gates. Run the focused new tests and record the pre-implementation failing result in your task report. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_28c3fe7a", "title": "Add generic semantic-retry compliance regression tests"} +{"board": "cassandra", "body": "Repair the same open Gitea atlas/cassandra PR #1 on branch handoff/generated-strategy-audit-20260813 in the existing workspace, starting from exact reviewed head 8c0118fc9d73878e6fc1e40a845529aa7647557a. Treat parent review t_b3a79a92 and its exact repro inputs as authoritative. Reproduce RED first, then make the smallest structural fixes for all four new findings: 1) anchor every spelled-number alternative as a complete token so fourteen cannot prove four, sixteen cannot prove six, tenth cannot prove ten, and ordinary words or card names containing one/two/three/four/five/six/seven/eight/nine/ten cannot create false cost claims; apply consistently to prose and table cost parsing. 2) Preserve correct proved decorated cells and ordinary prose such as often, attention, none, someone, component and Stoneforge while still blocking actual contradicted numeric and symbolic costs end-to-end. 3) Replace the any-pipe-is-structure heuristic with real Markdown table-row recognition so a pipe on an ordinary wrapped continuation line cannot split the claim, without joining headings, blank paragraphs, fences or genuine tables. 4) Make every table row whose width disagrees with its header fail closed, especially under-width rows that shift or omit a Casting Cost cell; preserve activation exemptions and proved well-formed rows. Add focused adversarial regressions for every exact reviewer input and nearby boundaries. Preserve all unrelated tracked/untracked files and the inherited audit-file hashes. Run the parent-specified focused compliance, semantic/entry, Program, deterministic acceptance and broad authoring suites; repeat old-head RED and neutered-gate non-vacuity; verify dynamic discovery, snapshot failure, index reuse/cardinality/runtime, py_compile, diff/LOC and Ruff if available. Commit and push one focused follow-up to the same branch. Verify local/remote branch, refs/pull/1/head and authenticated Gitea PR head all equal the new commit; PR must remain open, unmerged and undeployed. Do not merge or deploy. Status completed only after push and verification.", "expected_role": "implementation", "id": "t_2e07f244", "title": "Repair Cassandra PR 1 after 8c0118fc review"} +{"board": "cassandra", "body": "Perform a consequential independent read-only release review of private atlas/cassandra PR #1 at exact head 825d8a46b24eb1f1dc60d705afff82eefb37d512 against exact base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Do not trust prior done labels, prose, or claimed counts. Do not edit files, mutate Kanban, comment, commit, push, merge, deploy, or launch a certification job. Preserve all existing untracked files. Read AGENTS.md, the PR diff, and the COLLAB-013/COLLAB-014 sections; keep context bounded to relevant files and hunks unless evidence requires expansion. Review the full semantic repair compliance change for fail-open paths, false positives on supported or cautious prose, unsafe entry and exact-cost behavior, scope and metadata consistency, exception handling, regex/runtime risk, named-card production branches, 500-line constraints, regression coverage, and whether tests actually exercise the production path. Run small executable probes and focused tests needed to reproduce findings. Verify the PR body and exact topology. Return prioritized findings with exact file and line references and reproducible inputs. If no release-blocking defect is found, explicitly state merge readiness, residual risks, and the exact fresh persisted Lazav/Goreclaw certification still required after deployment. Do not call the live certification complete.", "expected_role": "implementation", "id": "t_35d7bb2e", "title": "Independently review Cassandra semantic-repair PR 1 for release"} +{"board": "cassandra", "body": "First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Using the Codex-authored certification implementation, execute the live v69 certification workflow against deployed 0.9.63 and collect durable evidence for COLLAB-001, COLLAB-002, and COLLAB-011. Verify persisted certification and semantic-review hashes can be retrieved and matched; run exact action-ranking replay and demonstrate equivalence; prove the recorded worker_app_version corresponds to the executing worker. Use small batches where games or provider work is involved, and classify zero-token provider failures as capacity blockers rather than product defects. Attach commands, raw outputs, artifact paths/hashes, model, provider, effort, and blockers; clearly distinguish successful proof from unavailable-capacity cases.", "expected_role": "implementation", "id": "t_3a16b9a2", "title": "Run live v69 certification and capture reproducible evidence"} +{"board": "cassandra", "body": "Perform a fresh consequential independent release review of private atlas/cassandra PR #1 at exact head e6bbd147aa710bc4406129bc25dacf094e01ab5c. Use the existing /opt/data/workspace/projects/cassandra-hermes-v69 checkout read-only. First verify local HEAD, canonical branch handoff/generated-strategy-audit-20260813, refs/pull/1/head, and the PR API head are byte-identical; verify the PR base from the API and review the entire current PR diff against that base, not only the latest commit and not any parent summary.\n\nDo not edit files, stage, commit, push, comment, merge, deploy, mutate Kanban other than normal completion, or alter the four inherited untracked files. Treat all prior test claims and done labels as untrusted evidence to reproduce where proportionate.\n\nConcentrate on release correctness around all historical review findings: Markdown table header/data/separator scanning and malformed fail-closed behavior; exact-cost grammar with N-mana source/rock/pip/resource prose versus true cost/mana-value/CMC/symbolic claims; caveat colon/semicolon/coordinating-clause/no-retroactive-waiver scoping; entry-hazard safety versus positive safe/caveated controls and the documented non-repair policy; snapshot reuse and adversarial runtime bounds; import/export/public API hygiene; production placement and regression coverage. Probe for bypasses, false positives, wording variants, multiline/table escapes, cardinality/performance regressions, and unintended changed semantics outside the target.\n\nIndependently run the final focused regression modules and broader strategy-authoring suite proportionate to this central parser change, plus deterministic acceptance, static/compile/diff checks available in the workspace. If a broad test failure is unrelated, reproduce and trace it before classifying it. Do not install new unreviewed dependencies or claim unavailable tools ran.\n\nPreserve and independently re-hash these untracked files without reading sensitive content into logs:\n- docs/generated_strategy_job_audit.md expected prefix 0e5c9160\n- scripts/audit_generated_strategy_job.py expected prefix 8259a7d8\n- tests/test_audit_generated_strategy_job.py expected prefix dbd2d776\n- uv.lock expected prefix f2d7ace7\n\nReturn a prioritized finding list with exact file/line references and reproducible fictional inputs for every defect. End with an explicit VERDICT: SHIP only if no blocking/material defect remains; otherwise VERDICT: BLOCK. State residual risks and exact tests. Keep PR #1 open, unmerged, and undeployed for human review.", "expected_role": "review", "id": "t_43d91361", "title": "Independent final review of Cassandra PR 1 at e6bbd147"} +{"board": "cassandra", "body": "Act as the consequential independent release reviewer for private atlas/cassandra PR 1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392, base main ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c, branch handoff/generated-strategy-audit-20260813. Read-only review only: do not edit, stage, commit, push, merge, deploy, comment on Gitea, or mutate Kanban outside this card. First verify the canonical origin, exact local and remote heads, PR state/topology, and clean tracked worktree. Preserve and exclude from the PR review these four inherited untracked files, verifying their hashes remain unchanged: docs/generated_strategy_job_audit.md 0e5c9160adbdad370a85b186fb52f8224fa708fa2c2ec935be46c35ba65f22ee; scripts/audit_generated_strategy_job.py 8259a7d8f2c5b2029fd0f8759419ecdcfa5495878bb5985daa3930f635a8ed46; tests/test_audit_generated_strategy_job.py dbd2d77691265c2fb74c7e6805ab5743a4d7bf3c715f9b4663d4cf19f1dea97c; uv.lock f2d7ace7c76ce3d910dc5d7c6b4c6345dd245068bc66981ae835772830469580. Independently inspect every production and test diff from origin/main through the exact head; do not trust prior done labels, docs, or test claims as truth. Reproduce or refute all six original review areas: caveat/unknown-prefix assertion scoping, common whole-deck closure grammar, colon-form exact costs, ambiguous card-name prefixes, activation versus casting tables, and explicit strategy antecedents. Adversarially audit the new occurrence-scoped table logic, including identical row text in activation and casting tables, mana-only activations, embedded ordinary exact-cost prose, casting-cost headers, valid Markdown tables with outer pipes omitted or columns reordered, misleading headers, fenced examples, and escaped or malformed cells. Distinguish safety fail-open defects from cautious false positives. Run the 61 focused compliance tests, 9 release-blocker tests, 7 semantic-review tests, 9 Program-quality tests, plus small production-path probes for any uncovered variants; do not duplicate the already-conclusive 618-test full suite unless focused evidence reveals a regression requiring it. Check changed-file LOC, static/diff hygiene, docs accuracy, and bounded runtime at intended input scale. Return a prioritized review with exact file/line evidence and reproducible inputs. Say merge-ready only if no material defect remains; otherwise state not merge-ready and enumerate the minimal general repairs and missing regressions. No certification or deployment claims.", "expected_role": "implementation", "id": "t_4620ed5c", "title": "Independently review Cassandra PR 1 at 49411de1"} +{"board": "cassandra", "body": "Finalize the independent read-only verdict for private atlas/cassandra PR #1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392 and base ee1a5f96. The technical audit evidence is preserved at /opt/data/home/.hermes/kanban/boards/cassandra/logs/t_4620ed5c.log. The later t_7aee0c6c attempt changed nothing and failed only because its strict JSON schema omitted findings from required; that platform defect is now fixed.\n\nYour assigned task is the review itself. If defects reproduce, complete the review with status=completed, put each defect/risk in findings, keep blockers empty, and state that the PR is not merge-ready. Use status=blocked only if an obstacle prevents you from performing this review.\n\nDo not edit, stage, commit, push, merge, deploy, comment in Gitea, or mutate other Kanban cards. Revalidate:\n- canonical origin, exact local/remote/PR heads, tracked-clean state, and the four inherited untracked hashes;\n- exact source/line causes and representative production-path inputs from the prior audit;\n- table handling for omitted outer pipes, reordered columns, misleading headers containing Activation Cost, and a row containing both Activation Cost and Casting Cost;\n- unsupported exact-cost grammar such as “has a mana cost of five mana”, “is cast for five mana”, “CMC is five”, plus unknown-clause scoping across although/while versus but/semicolon;\n- the explicit strategy antecedent boundary “Our strategy centers on curving out, and no card in the deck changes it”;\n- cautious false positives separately (escaped activation cells, Cost to Activate, malformed cells).\n\nRerun small executable probes and one representative focused regression command; do not repeat the 618-test baseline or all expensive gates. Confirm prior green suite/hygiene evidence from the preserved log as prior evidence, not newly run. Return a prioritized release verdict with exact file/line references and reproducible inputs.", "expected_role": "review", "id": "t_4a0eb5f2", "title": "Finalize Cassandra PR 1 independent verdict under v4 contract"} +{"board": "cassandra", "body": "Independently review private atlas/cassandra PR #1 at exact head 4a0540501e3fbee1d931af042942f0dc30d429e0 against base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Read-only consequential final release review: do not edit files, mutate Kanban beyond your own task result, comment, commit, push, merge, deploy, or certify live guides. Verify the remote/PR head first. Inspect the entire diff and every changed production/test/documentation file. Do not trust prior done labels or claimed test evidence as truth; use focused read-only probes/tests as needed. Prioritize fail-open table parsing and per-cell scoping, exact-cost grammar coverage/false positives, caveat clause boundaries, whole-deck closure, anaphor/antecedent precedence, malformed input, ReDoS/runtime bounds, production-path placement, documentation accuracy, and missing adversarial tests. Report prioritized findings with exact file/line evidence and reproducible inputs. If no material defect exists, explicitly state residual risks and why evidence is sufficient. Preserve all inherited untracked files byte-for-byte.", "expected_role": "implementation", "id": "t_53cd8b4d", "title": "Independent final review of Cassandra PR 1 repair"} +{"board": "cassandra", "body": "Research and document the exact current path to perform (after a successful certified guide exists) the remaining scorecard gates in docs/handoff_hermes_codex_20260809.md: retrieve full job export and saved guide safely from the production PostgreSQL generation queue/strategy store; create a generic saved-Program action-ranking counterfactual suite covering each lane; run the audit harness; separately queue small Forge measurements. Inspect source, deployment/IaC, and existing reports only; do not change code, queue jobs, or expose credentials. Return exact source locations, operational commands/procedures that have evidence in repo, and blockers/unknowns. It must distinguish what can be automated locally vs requires live job artifacts.", "expected_role": "implementation", "id": "t_551203be", "title": "Map certified-job retrieval and action-ranking proof workflow"} +{"board": "cassandra", "body": "Using the completed implementation and validation evidence in /opt/data/workspace/projects/cassandra-hermes-v69, update docs/agent_collaboration.md and DEVLOG with the semantic-retry correction, the generic regression scenarios, the bounded/fail-closed behavior, and exact focused, broad-generator, and 9/9 acceptance results. Inspect git diff/status and prepare a review-blocked handoff stating changed files, exact test results/commands, and concise diff evidence that no named-card production branch was introduced. Leave all changes uncommitted; do not push or deploy.", "expected_role": "implementation", "id": "t_5b926b7c", "title": "Document results and prepare review-blocked handoff"} +{"board": "cassandra", "body": "Finalize the independent read-only verdict for private atlas/cassandra PR #1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392 and base ee1a5f96. The previous audit card t_4620ed5c completed its technical review but was parked by an old result-contract ambiguity; its full worker log is authoritative audit input at /opt/data/home/.hermes/kanban/boards/cassandra/logs/t_4620ed5c.log.\n\nYour assigned task is the review itself. If defects reproduce, complete the review with status=completed, put each defect/risk in findings, keep blockers empty, and state that the PR is not merge-ready. Use status=blocked only if an obstacle prevents you from performing this review.\n\nDo not edit, stage, commit, push, merge, deploy, comment in Gitea, or mutate other Kanban cards. Revalidate:\n- canonical origin, exact local/remote/PR heads, tracked-clean state, and the four inherited untracked hashes;\n- exact source/line causes and representative production-path inputs from the prior audit;\n- table handling for omitted outer pipes, reordered columns, misleading headers containing Activation Cost, and a row containing both Activation Cost and Casting Cost;\n- unsupported exact-cost grammar such as “has a mana cost of five mana”, “is cast for five mana”, “CMC is five”, plus unknown-clause scoping across although/while versus but/semicolon;\n- the explicit strategy antecedent boundary such as “Our strategy centers on curving out, and no card in the deck changes it”;\n- cautious false positives separately (escaped activation cells, Cost to Activate, malformed cells).\n\nRerun small executable probes and a representative focused regression command; do not repeat the 618-test baseline or all expensive gates. Confirm prior green suite/hygiene evidence from the preserved log, but do not claim it as newly run. Return a prioritized release verdict with exact file/line references and reproducible inputs.", "expected_role": "review", "id": "t_7aee0c6c", "title": "Finalize Cassandra PR 1 independent verdict at 49411de1"} +{"board": "cassandra", "body": "Act as the sole implementation owner in /opt/data/workspace/projects/cassandra-hermes-v69. Consume the architecture findings and RED regressions from parent tasks, then implement the smallest robust production fix that closes all four bypasses: reject unsupported exact commander-cost assertions including symbolic and word-number forms; detect unqualified exhaustive whole-deck paraphrases even when padded with supplied-metadata hedges; rerun semantic compliance after deterministic claim repair before accepting repaired output; and distinguish explicitly negated/caveated universal statements so legitimate polarity-aware wording is permitted. Maintain fail-closed behavior for entry hazards, unknown cards, legal targets, and provenance. Run the new regressions and relevant nearby semantic/entry suites, iterating until GREEN. Do not run the expensive full broad suite or downstream 9/9 acceptance suite. Do not commit, push, deploy, or delegate final completion to a reviewer. Return the final structured work result yourself, including changed files, test commands/results, and any remaining risks.", "expected_role": "implementation", "id": "t_92e8e57c", "title": "Implement and validate semantic-retry compliance hardening"} +{"board": "cassandra", "body": "Resume from clean main at 382b2c1c and deployed 0.9.63. Read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md first. Finish live v69 certification evidence for COLLAB-001, COLLAB-002, and COLLAB-011: persisted certification and semantic-review hashes, exact action-ranking replay, and worker_app_version proof. Preserve cross-provider separation: Codex authors or implements; Claude performs independent high-tier review; Hermes synthesizes. Only after certification is clean, unblock the 4+ decided-game measurement set. Respect small game batches and treat zero-token provider errors as capacity, not defects. Record commands, artifacts, model, provider, effort, and blockers on this task.", "expected_role": "implementation", "id": "t_97df755a", "title": "Certify and prove the deployed Lazav v69 Program"} +{"board": "cassandra", "body": "Repair atlas/cassandra PR #1 after the independent BLOCK review t_43d91361. Work in the existing /opt/data/workspace/projects/cassandra-hermes-v69 checkout on branch handoff/generated-strategy-audit-20260813, starting at exact head e6bbd147aa710bc4406129bc25dacf094e01ab5c. First verify local HEAD, canonical remote branch, refs/pull/1/head, and PR API head/base. Preserve all unrelated and untracked work; re-hash these four inherited files before and after without logging contents: docs/generated_strategy_job_audit.md prefix 0e5c9160, scripts/audit_generated_strategy_job.py 8259a7d8, tests/test_audit_generated_strategy_job.py dbd2d776, uv.lock f2d7ace7. Do not merge or deploy.\n\nTreat the exact durable completed result of review t_43d91361 as authoritative findings to reproduce, not as implementation instructions to copy blindly. Add RED tests first and make the smallest structural fixes for all four material findings:\n1. Cost claims split by ordinary Markdown soft-wrap newlines currently bypass every grammar family. Scan paragraph-continuation whitespace without joining headings, blank-separated paragraphs, or table structure; test symbolic, CMC/mana-value, and numeric claims end-to-end through the accepting-critic/reused-chunk path.\n2. Casting-cost table cells only block when the value is the entire cell. Parse the already escape-aware cell value semantically so leading/trailing prose, parentheticals, reductions notes, numeric mana text, and escaped-pipe suffixes cannot hide a contradicted cost; preserve activation-cost table exemptions and fail closed on malformed tables.\n3. The resource-noun tail heuristic incorrectly cancels claims that explicitly say N mana when a later adjunct mentions turns/lands/creatures/cards/opponents. Explicit mana units must remain cost assertions; preserve positive non-cost controls for mana sources, pips, rocks, dorks, life, and other activation/resource prose.\n4. Safe conditional entry-hazard guidance such as “Only cast Catastrophic Colossus with two expendable bodies already on board” and “Cast ... only when you can pay its entry sacrifice” is a pre-existing classifier false positive that this PR newly turns into an unrecoverable hard failure on accepted/reused chunks. Refine the classifier or compliance routing so correct protective conditions survive while truly unsafe “without paying/ignoring the sacrifice/entry drawback” guidance still blocks. Ensure a repair path exists where appropriate without weakening fail-closed entry-hazard enforcement.\n\nAlso assess and fix the low adjacent-table inheritance defect if it can be done safely in the same parser change; do not broaden lexical grammar merely to chase the documented idiom boundary. Remove or replace the _NON_CASE_MODULES support-module exclusion if it can silently hide future tests. Do not add grammar-only patches for structural escapes.\n\nAcceptance: reproduce each reported case RED before implementation; add adversarial positive/negative controls; run the seven focused compliance modules (prior baseline 90), Program quality (9), semantic-review plus entry-hazard modules (55), deterministic acceptance (9/9), and the broad tests.test_strategy_authoring_ai suite (647 baseline). Repeat vacuity/structural probes proportionately, py_compile/diff/LOC/ruff if available, and runtime/cardinality bounds including aggregate cost-index reuse if changed. Keep every source/test file below repository limits. Commit and push one focused repair to the same branch so PR #1 advances; verify remote branch/pull ref/API head equal the new commit and PR remains open, unmerged, undeployed. Report exact changed files/tests and any unavailable tools. Status is completed only after push and verification; use incomplete if tests still run.", "expected_role": "implementation", "id": "t_9826e3ce", "title": "Repair Cassandra PR 1 structural compliance escapes after review 53"} +{"board": "cassandra", "body": "Repair the four reproduced semantic-retry compliance failures in /opt/data/workspace/projects/cassandra-hermes-v69 using TDD and generic fictional metadata only. Add regressions for: (1) unsupported commander exact-cost phrasing such as requires {3}{G}{G} to cast and need five mana to cast; (2) exhaustive whole-deck paraphrases such as every card in the deck leaves the commander cost unchanged, including superficial supplied-metadata hedges that still quantify over the whole deck; (3) recomputing semantic compliance after deterministic claim repair so a corrected commander name cannot introduce an unchecked exact-cost claim; and (4) polarity-aware handling that permits explicit caveats/negations such as do not assume no card in the whole deck changes the cost. Preserve fail-closed entry-hazard, unknown-card, legal-target, and provenance behavior. Run the new RED/GREEN regressions plus nearby semantic/entry suites. Do not run the expensive full broad suite here; the downstream acceptance task will run focused, broad, and 9/9 after this fix. Do not commit, push, or deploy. Do not delegate Kanban task completion to a review child: delegated reviewers may only return findings to the foreground worker; only the foreground worker may produce the final structured task result.", "expected_role": "implementation", "id": "t_abafcb10", "title": "Close reproduced semantic-retry compliance bypasses"} +{"board": "cassandra", "body": "Independently review the untracked generated Strategy audit harness in `/opt/data/workspace/projects/cassandra-hermes-v69`: `scripts/audit_generated_strategy_job.py`, `tests/test_audit_generated_strategy_job.py`, and `docs/generated_strategy_job_audit.md`. Review against docs/handoff_hermes_codex_20260809.md acceptance scorecard 5/6 and actual repository contracts. Identify correctness gaps, especially dynamic semantic-review batch requirements, exact Program extraction/hashing/equality, ranking-suite validity/coverage, and whether an audit can falsely return OK. Do not edit files. Record an evidence-backed verdict with paths/lines and exact test commands; if sound, say so. This must not claim live certification proof.", "expected_role": "review", "id": "t_aefd1147", "title": "Independently review untracked generated-strategy audit harness"} +{"board": "cassandra", "body": "Implement the current handoff critical-path correction on `/opt/data/workspace/projects/cassandra-hermes-v69`, following AGENTS.md and docs/agent_collaboration.md. Current deployed 0.9.65 failures: Lazav retry 8 job `4691c3a7-357b-43a3-88ec-d8a923d9708c` failed because semantic repair retained/introduced unsafe Phyrexian Dreadnought and Phage entry routes; Goreclaw retry 7 `982e698d-17a2-4515-9263-13267566626d` failed because repair retained unsupported exact commander cost and whole-deck closure claims with incomplete facts. Work strictly TDD: reproduce generic fictional metadata-driven failures first, see the new tests fail, then implement minimal generic bounded semantic-repair compliance. Preserve fail-closed entry hazard, unknown-card, legal-target, and provenance gates; no production named-card branches. Run focused and broad generator suites plus 9/9 acceptance, update docs/agent_collaboration.md + DEVLOG with exact results. Do not commit/push/deploy; leave reviewable diff. Block for review with changed files/test result/diff evidence. ", "expected_role": "implementation", "id": "t_af7738f5", "title": "Repair semantic-retry compliance for live Lazav and Goreclaw failures"} +{"board": "cassandra", "body": "Review the Codex implementation/evidence and the independent Claude review under docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Produce the Hermes synthesis decision for live v69 certification of COLLAB-001, COLLAB-002, and COLLAB-011: reconcile discrepancies, validate that cross-provider separation was maintained, enumerate final artifact hashes and locations, and state a clear clean/pass, conditional, or blocked verdict. Authorize the decided-game measurement set only on a clean certification verdict; otherwise specify precise remediation and retain the gate. Record commands consulted, artifacts, model/provider/effort, and blockers.", "expected_role": "implementation", "id": "t_b0a1b0b5", "title": "Synthesize certification decision and authorize measurement gate"} +{"board": "cassandra", "body": "Perform a fresh consequential independent read-only release review of private Gitea atlas/cassandra PR #1 at exact head 8c0118fc9d73878e6fc1e40a845529aa7647557a against exact API base and merge-base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. This is scm.bstein.dev Gitea, not GitHub: do not use GitHub/gh skills or treat gh auth as relevant. The supplied workspace is a clean detached worktree. Do not edit files, commit, push, comment, approve, merge, deploy, mutate Kanban other than completing this assigned card, create pods, or expose credentials. Verify local head, canonical branch, pull ref, authenticated Gitea API head/base/state and merge-base before trusting the diff. Inspect the entire current base-to-head PR diff, with particular scrutiny of commit 8c0118fc and every reviewer finding from parent t_43d91361. Independently reproduce: ordinary Markdown soft-wrap bypasses across symbolic/numeric/CMC/mana-value grammar without joining headings, blank paragraphs or tables; decorated casting cells with prose, parentheses, reductions, numeric mana and escaped pipes while activation/proved costs remain accepted and malformed/adjacent tables fail closed; explicit N-mana claims with later turn/land/creature/card/opponent adjuncts while resource/pip/rock/dork/life/activation prose stays non-cost; safe two-body or pay-entry-sacrifice conditions through accepting critic/reused chunks while without-paying/ignore drawback remains blocked and repair path works. Check Oracle/snapshot consistency, exception behavior, scope/caveat boundaries, false positives and any nearby wording variants not in tests. Verify dynamic support-module discovery and no hidden tests. Reproduce old-head RED and neutered-gate non-vacuity. Run the focused compliance, semantic-review/entry, Program quality, deterministic acceptance, broad authoring suite, syntax/diff/LOC and runtime/cardinality/index-reuse checks independently. Ruff is optional only if truly unavailable. Preserve all files; use external temp state. Return findings first with exact file/line and reproducible inputs. SHIP only with no material defect and full evidence; otherwise BLOCK. Human review remains required and nothing may be merged/deployed.", "expected_role": "review", "id": "t_b3a79a92", "title": "Independent release review Cassandra PR 1 at 8c0118fc"} +{"board": "cassandra", "body": "Prepare one reviewable Gitea pull request for the already committed Cassandra semantic-repair compliance correction. Read AGENTS.md, COLLAB-013, and COLLAB-014 first. Preserve every existing untracked file exactly: docs/generated_strategy_job_audit.md, scripts/audit_generated_strategy_job.py, tests/test_audit_generated_strategy_job.py, and uv.lock. Do not edit source files, create commits, push, merge, deploy, launch a certification job, or mutate Kanban beyond this card. Verify canonical base atlas/cassandra main is ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c, head branch handoff/generated-strategy-audit-20260813 is 825d8a46b24eb1f1dc60d705afff82eefb37d512, the branch is exactly one commit ahead and zero behind, no open PR already exists for this head, and git diff --check passes. Re-run the three focused unittest modules for semantic retry compliance. If every check passes, create exactly one pull request in private atlas/cassandra using the credential-safe gitea_api.py helper and shell-safe repeated --field arguments. The PR body must record exact commit SHAs and focused, broad, acceptance, vacuity, neutered-gate, py_compile, LOC, ReDoS, and diff-check evidence from COLLAB-014, and must state that a fresh persisted certification job and independent release review remain required before merge or deployment. Stop after verifying the PR exists and is open. Report the PR number and URL without credentials.", "expected_role": "implementation", "id": "t_b87ef6e5", "title": "Prepare Cassandra semantic-repair compliance pull request"} +{"board": "cassandra", "body": "Proceed only if the Hermes synthesis artifact explicitly records a clean certification verdict. First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md, then execute and document a measurement set containing at least four decided games using the certified v69/deployed 0.9.63 path. Keep game batches small, preserve all version and provenance metadata, and report per-game decision/outcome plus aggregate measurements. Treat zero-token provider errors as capacity events: log them, retry only within documented limits, and do not label them as defects. Attach commands, artifacts, model/provider/effort, and blockers to the task.", "expected_role": "implementation", "id": "t_cc83234b", "title": "Run gated 4+ decided-game v69 measurement set"} +{"board": "cassandra", "body": "Starting from clean main at 382b2c1c, first read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. As the Codex author, inspect the deployed 0.9.63/v69 certification path and implement or repair all missing support for COLLAB-001, COLLAB-002, and COLLAB-011: persisted certification hashes, persisted semantic-review hashes, deterministic exact action-ranking replay, and worker_app_version provenance. Preserve cross-provider separation; do not represent Claude review as Codex review. Add or update focused tests and produce commit/diff references plus artifact locations. Record every command, artifact, model/provider/effort metadata, and blockers in the task handoff.", "expected_role": "implementation", "id": "t_cda55d42", "title": "Implement persisted v69 certification and replay evidence"} +{"board": "cassandra", "body": "Repair only the independently reproduced release blockers on private atlas/cassandra PR #1, exact current head 825d8a46b24eb1f1dc60d705afff82eefb37d512 on handoff/generated-strategy-audit-20260813. Work test-first in the existing linked checkout. Preserve the four inherited untracked files byte-for-byte and do not delete, stage, or rewrite them. Before production edits, add RED generalized regressions through the real _review_and_repair_semantics path for: sentence-scoped caveats so an unrelated trailing caution cannot waive an unsafe closure; unrelated unknown prefixes cannot waive a later exact-cost assertion; common closure forms including isn't and deck has no card; colon-form exact casting cost; ambiguous comma-name prefixes must not borrow a different card's facts; Markdown activation-cost tables such as Forest | {T} | Add {G}. must not be treated as casting-cost claims; and explicit strategy antecedents such as The plan is to curve out, and no card in the deck changes it must remain allowed without reopening unsafe pronoun closure. Use fictional/generic fixtures, not named-card patches. Split the 515-line changed test module into meaningful <=500-line modules and keep every changed production and test file <=500 lines. Update stale COLLAB-013/COLLAB-014/DEVLOG wording so it accurately describes the committed PR state. Keep the gate wired into the production repair path and preserve fail-closed exception behavior. Run the focused compliance suites, section semantic-review tests, new production-path adversarial probes, LOC checks that explicitly include changed tests, py_compile, git diff --check, and a bounded regex/runtime probe; expand to relevant broader tests if changes touch shared parsing. Commit and push the corrections to the same PR branch and verify exact remote topology/PR mergeability, but do not merge, deploy, launch certification, or claim Lazav/Goreclaw certification. Return exact files, red-to-green evidence, test counts/durations, new head SHA, preserved-untracked verification, and any residual risks.", "expected_role": "implementation", "id": "t_d248853a", "title": "Repair Cassandra PR 1 independent-review blockers"} +{"board": "cassandra", "body": "Fresh independent release review of atlas/cassandra PR #1 at exact head 14c07111b6ed8d7fc529362eb34fa0afa0694325 against API-confirmed base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Use Claude Code subscription OAuth only through cli-claude-xhigh; no metered API keys. Work read-only in a new detached worktree; do not edit implementation state, commit, push, comment, merge, deploy, or mutate external state. Treat parent t_15ed4f67 results as claims to verify, not truth. Pin PYTHONPATH to the detached tree for every test and print/assert imported module paths to avoid the editable-install trap. Reproduce every finding from t_15f2a75c end-to-end through section_pipeline: underscore-emphasized digits/words; realistic Commander/Cards/Spell Name and non-adjacent cost headers; claims in card cells; pipe-bearing prose/list rows adjacent to casting and activation tables; every affected ordinal; cardinals above nineteen and one-hundred prefix cases. Audit compositional number grammar for false proof/rejection across 0..hundreds, hyphenation, conjunctions, ordinals/cardinals, punctuation/Markdown and ordinary words/names/negated/cautious prose. Audit table scope for unknown headers, ambiguous/cardless/multiple-card rows, malformed widths, escaped pipes, GFM continuations, headings/fences/blank boundaries and occurrence-local activation exemptions. Recheck snapshot/exception fail-closed, one index/snapshot per call, deterministic cardinality, bounded runtime and no pathological regex behavior. Run focused compliance, semantic/entry, Program, deterministic acceptance, full dynamic discovery, exact old-head RED and neutered-gate RED with pinned module paths. Verify local/remote/pull/API exact head and PR open/unmerged/undeployed. Return strict findings-first SHIP/BLOCK with exact file/line and executable minimal repro; residual risks if SHIP. Remove temporary review resources and stop read-only.", "expected_role": "review", "id": "t_d3ef05d3", "title": "Independent release review of Cassandra 14c07111"} +{"board": "cassandra", "body": "Repair every defect found by independent review task t_53cd8b4d on private atlas/cassandra PR #1, starting from exact head 4a0540501e3fbee1d931af042942f0dc30d429e0. Work only in the existing /opt/data/workspace/projects/cassandra-hermes-v69 directory and existing handoff/generated-strategy-audit-20260813 branch. Canonical remote is atlas/cassandra. Keep the PR open for human review; do not merge, deploy, or comment externally.\n\nPreserve these four inherited untracked files byte-for-byte and never stage or commit them:\n- docs/generated_strategy_job_audit.md (sha256 prefix 0e5c9160)\n- scripts/audit_generated_strategy_job.py (8259a7d8)\n- tests/test_audit_generated_strategy_job.py (dbd2d776)\n- uv.lock (f2d7ace7)\n\nBefore production edits, add and run focused fictional RED regressions that demonstrate each failure/acceptance boundary below. Then implement general bounded fixes, not phrase-specific exceptions:\n1. Fail-open Markdown header cells: a factual false cost claim in any table header cell must be scanned and blocked, while pure header labels and separator rows remain valid. Preserve optional outer pipes, escaped pipes, reordered columns, mixed activation/casting scoping, and malformed-input fail-closed behavior.\n2. False-positive N-mana resource prose: natural descriptions such as “needs two mana sources on board” and “requires three mana rocks” must not be treated as exact casting-cost claims. Correct claims like “costs two mana” and all existing symbolic/CMC/mana-value forms must remain enforced.\n3. Extend non-cost resource-noun handling for source(s), pip(s), and defensibly adjacent common resource nouns without creating an escape hatch for actual exact-cost claims. Cover “requires one white mana source” and “costs two pips of black.”\n4. Caveat colon scope: “Do not assume the following: Grave Recall costs {9}{B}.” must be accepted as caveated, without allowing a colon in unrelated prose to waive later claims. Preserve semicolon/coordinating-clause boundaries and no retroactive waiver.\n5. Hygiene: remove the unused mentions_card and claim_clauses imports; remove or make purposeful the uncalled public card_fact helper; make __all__ accurately export cross-module public functions. Since Ruff is absent, run a deterministic static unused-import/export fallback and document the explicit Ruff skip honestly.\n6. Add the missing positive control around the deliberate unsafe-entry behavior. Preserve the documented strict release choice that compliance blockers are not mechanically repaired unless evidence proves this causes an unintended safety or correctness defect; demonstrate that safe/caveated entry prose is accepted and unsafe entry guidance is rejected. Do not silently change the policy.\n7. Assess the measured ~0.57s compliance-call cost. Optimize repeated snapshot/index construction only if a small safe change can be proven equivalent; otherwise record it as residual performance risk rather than expanding scope.\n\nRun the full focused compliance/review suites and any broader grammar/strategy suite proportionate to this central parsing change. Confirm no ReDoS/runtime regression with bounded adversarial probes. Verify local HEAD, canonical remote branch, and PR #1 head are byte-identical after commit/push. Report exact commit, changed files, RED-before/GREEN-after evidence, tests, explicit skips, residual risks, PR state, and inherited-file hashes.", "expected_role": "implementation", "id": "t_d5274164", "title": "Repair Cassandra PR 1 findings from independent final review"} +{"board": "cassandra", "body": "Runtime-only acceptance canary for the external Kanban goal loop. Do not run tools or commands, read or edit files, access the network, or mutate any external state. On the first provider turn, return the required structured result with status=incomplete, summary exactly PHASE_ONE_INCOMPLETE, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary phase one. On the resumed provider turn, return status=completed, summary exactly GOAL_LOOP_OK, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary: passed. Do not return completed on the first turn. Completion is valid only after Hermes rejects the first report and resumes this same card.", "expected_role": "implementation", "id": "t_dd6dc0a6", "title": "Goal-loop completion canary v2"} +{"board": "cassandra", "body": "Continue the preserved Cassandra PR 1 repair from task t_d248853a in the existing workspace. Do not redo completed work. Exact workspace: /opt/data/workspace/projects/cassandra-hermes-v69. Exact branch at start should be handoff/generated-strategy-audit-20260813 with HEAD b42ebdd600710d3cc2e61290d5fb08f1c23518e1 tracking origin. Preserve all unrelated and pre-existing work. The six intended uncommitted follow-up files are docs/agent_collaboration.md, docs/history/DEVLOG.md, semantic_repair_costs.py, semantic_repair_quality.py, semantic_repair_scope.py, and semantic_retry_compliance_release_blockers.py. Preserve these four inherited untracked files byte-for-byte with hashes: docs/generated_strategy_job_audit.md 0e5c9160adbdad370a85b186fb52f8224fa708fa2c2ec935be46c35ba65f22ee; scripts/audit_generated_strategy_job.py 8259a7d8f2c5b2029fd0f8759419ecdcfa5495878bb5985daa3930f635a8ed46; tests/test_audit_generated_strategy_job.py dbd2d77691265c2fb74c7e6805ab5743a4d7bf3c715f9b4663d4cf19f1dea97c; uv.lock f2d7ace7c76ce3d910dc5d7c6b4c6345dd245068bc66981ae835772830469580. The follow-up generalizes markdown table handling: activation-cost tables including mana-only costs must not be mistaken for casting costs; casting-cost tables and embedded ordinary exact-cost prose must still block. Prior evidence is 8/8 new release blocker tests, 60 focused compliance, 7 semantic-review, 9 program-quality, pycompile, changed-file max 438 LOC, production LOC/docs/structure/diff/bounded-runtime checks green. A prior full 616 test run passed before this table-header follow-up; the follow-up full rerun was interrupted and is not evidence. Inspect the diff first. Run the focused gates and exactly one conclusive full tests.test_strategy_authoring_ai run with enough time. If any failure occurs, repair generally and rerun affected gates. Then rerun static/LOC/docs/runtime checks, verify the four hashes, commit only the intended six tracked follow-up files, push the same branch to origin, and verify the remote exact head and PR 1 topology. Do not add or commit the four inherited untracked files. Do not merge, deploy, certify, mutate Kanban outside this card, or claim completion while tests or a push are still in progress. Completion requires a pushed follow-up commit plus exact-head and test evidence.", "expected_role": "implementation", "id": "t_df41be0d", "title": "Finalize Cassandra PR 1 activation-table repair"} +{"board": "cassandra", "body": "The full 234-module regression run under an ambient Hermes Agent environment exposed two unrelated test-isolation failures: tests.playtest_cases.manifest.PlaytestManifestTests.test_codex_decision_config_builds_worker_command assumes a `codex` binary exists instead of mocking discovery, and tests.strategy_authoring_ai_cases.claude_cli_provider.ClaudeCliExecutionTests.test_api_key_secret_maps_to_api_key_env inherits CLAUDE_CODE_OAUTH_TOKEN and dumps the whole environment on assertion failure. Harden both tests/fixtures so they are deterministic under agent credentials and never render the full environment or secret values in failure output. Verify both focused tests plus scripts/parallel_unittest.py. Do not change provider semantics merely to satisfy the tests.", "expected_role": "implementation", "id": "t_e822d35d", "title": "Harden full-suite tool and credential environment isolation"} +{"board": "cassandra", "body": "After the regression tests are in place, implement the minimal generic production change needed for the new semantic-retry compliance tests to pass. Work in /opt/data/workspace/projects/cassandra-hermes-v69 and follow AGENTS.md plus docs/agent_collaboration.md. Make repair decisions from metadata, supported facts, and provenance only: reject unsafe entry routes, reject unknown or illegal targets, and remove/reject unsupported exact cost and whole-deck closure assertions when evidence is incomplete. Preserve existing fail-closed entry-hazard, unknown-card, legal-target, and provenance gates; do not weaken them. Do not add any named-card, named-commander, job-ID, or production-fixture branches. Keep the patch narrowly scoped, run the focused tests until green, and report changed files plus test output. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_ef36526b", "title": "Implement bounded generic semantic-repair compliance"} +{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, inspect the semantic-retry pipeline, deterministic claim-repair flow, entry-hazard checks, and existing semantic/entry tests. Produce concise findings for the implementation owner identifying the exact functions, ordering constraints, existing test conventions, and safe extension points for: unsupported exact commander-cost phrasing; whole-deck universal paraphrases and metadata hedges; re-checking semantic compliance after deterministic repair; and polarity-aware caveat/negation handling. Do not modify production behavior, commit, push, deploy, or return a final task-completion result; return implementation findings only.", "expected_role": "implementation", "id": "t_fdc1a23e", "title": "Map semantic-retry compliance decision paths and repair hazards"} +{"board": "soteria", "body": "Perform a fresh independent read-only release review of atlas/soteria PR #11. Canonical implementation head claimed by the implementer is 0143d47; verify the live PR head/base/state and review the entire base-to-head diff without trusting its result. Reproduce issue #9's original go:S3776/metrics complexity condition and prove the boolGauge extraction preserves every metric name, label, 1/0 health value, concurrency behavior, and failure path. Look for semantic drift, races, NaN/type behavior, false confidence from narrow tests, docs/LOC/style regressions, and whether the Sonar claim is actually supported. Run focused tests, go test -race ./..., go vet, gofmt, project check/quality commands, and inspect live PR checks read-only where available. Do not edit any file, comment, commit, push, merge, close the issue, deploy, or mutate Kanban outside returning this task result. Return completed with a clear SHIP or BLOCK verdict in summary/findings; review findings are findings, not task blockers. Preserve all inherited/untracked files and prove the review worktree remains clean.", "expected_role": "review", "id": "t_b1d2cd0c", "title": "Independent release review of Soteria PR 11"} +{"board": "soteria", "body": "Resolve private atlas/soteria issue #9 (`[hermes] soteria: go:S3776`) from exact main head 47205db5b2e8041bac6a06a78f94fd3662134fc8. Work only in the isolated Hermes worktree derived from /opt/data/workspace/projects/soteria. Inspect the issue and current code independently; the issue's suggested boolGauge extraction is a hypothesis, not trusted implementation truth.\n\nAcceptance:\n- Add a focused RED-before/GREEN-after test proving the healthy PVC metric remains exactly 1 for true and 0 for false, with labels/metric identity unchanged.\n- Reduce RecordInventory cognitive complexity at internal/server/metrics.go:133 without behavior drift, phrase-specific workarounds, or unrelated refactors. If a helper is appropriate, keep it domain-named, small, documented only where useful, and tested.\n- Run targeted tests, the relevant full Go suite/race/static checks, formatting, Sonar/quality checks available locally, and any repository-specified validation. Record exact skips honestly.\n- Preserve unrelated files and verify the issue still applies at the exact starting head before editing.\n- Commit a focused branch, push to canonical atlas/soteria, and open a non-draft PR linked to issue #9 for human review. Do not merge, deploy, close the issue manually, or mutate cluster state.\n- Never expose or commit credentials. Use the runtime GIT_ASKPASS only.\n\nReport exact commit, branch, PR URL, changed files, RED/GREEN evidence, tests, skips, residual risks, and confirmation that main/issue were not modified directly.", "expected_role": "implementation", "id": "t_f1593f8c", "title": "Resolve Soteria issue 9 metrics complexity"} +{"board": "titan-iac", "body": "Perform a fresh independent consequential review of private atlas/titan-iac PR #13 at exact head 4f8dcfbbf748e88726912bef1ae32201e1e640a6 against base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Work read-only in the supplied worktree. First verify local HEAD, canonical remote branch wt/t_cca008de, refs/pull/13/head, and PR API head/base exactly match. Review the complete PR diff, not prior summaries. Do not edit, stage, commit, push, comment, merge, deploy, mutate Kanban beyond normal completion, create pods, build or publish images, or expose credentials.\n\nReview the Dockerfile source-patch anchors against the pinned upstream /opt/hermes implementation and prove they fail closed under upstream drift. Reproduce the current create_task(initial_status=blocked) defect and verify the extracted patched tree makes explicit initial blocks sticky, explicit unblock reversible, dependency promotion intact, and circuit-breaker blocks non-promotable. Check whether the added build-time regression actually runs in the image build context and catches both false negatives and false positives. Review concurrency 4->2, CPU cap 3->2, and enableServiceLinks=false for resource/scheduling correctness and for any runtime dependency on generated service environment variables. Check tests, style, security, secret hygiene, line limits, kustomize render, client dry-run, and diff hygiene. Distinguish ambient live-environment test failures from PR regressions by reproducing against the base where needed. The absence of a safe image builder is a known deployment blocker, not permission to improvise privileged Docker.\n\nReturn prioritized findings with exact file/line evidence and reproducible cases. End with VERDICT: SHIP only if the code/config PR is reviewable without a material defect; otherwise VERDICT: BLOCK. Keep PR #13 open, unmerged, and undeployed for human review.", "expected_role": "review", "id": "t_06771c95", "title": "Independent review of Hermes platform PR 13 at 4f8dcfbb"} +{"board": "titan-iac", "body": "Independent read-only xhigh security audit of the full-handoff acceptance harness produced by parent task t_6496c271.\n\nSETUP (do this first, trust nothing claimed): resolve the exact draft PR, head SHA, and base SHA dynamically rather than from any hardcoded number — list open/draft PRs for the repo, identify the one introducing the full-handoff acceptance harness, and record pr_number, head_sha, base_sha, plus the resolved changed-file list. Pin every finding to that head SHA. Read the full files, not just the diff hunks.\n\nSCOPE:\n1. Command construction: enumerate every place the harness builds a subprocess/exec/HTTP invocation. Confirm no shell=True or string interpolation of untrusted values, no injection via branch/PR/namespace/pod/image names, and that argument lists are built from validated inputs.\n2. Redaction BEFORE execution and BEFORE logging: verify the command line as logged/echoed is redacted prior to being run, not after, and that redaction lives at a single choke point rather than being reapplied per call site.\n3. Sanitization coverage: environment variables passed to children, HTTP headers, request/response bodies, captured stdout, captured stderr, and exception messages/tracebacks. Every one of these paths must be scrubbed. Look specifically for unscrubbed re-raises, repr()/f-string of a response or exception, and error paths that bypass the sanitizer.\n4. Credential containment: prove the harness cannot read or print Vault tokens/paths, cloud provider credentials, or SCM tokens. Trace every environment read and every credential-shaped file/secret access. Construct a concrete test that injects fake secrets into env/headers/responses and asserts they never appear in stdout, stderr, logs, or the JSON artifact.\n5. Ephemeral resource arming/cleanup and protected-branch rejection: verify arming is explicit and opt-in, cleanup is guaranteed on success, failure, timeout, and signal paths (no leaks), and that operations against protected branches are rejected early and unconditionally.\n\nCONSTRAINTS: keep the worktree pristine — no edits, no PR comments, no commit/push/merge/deploy/reconcile, no live mutations of any cluster or provider. Reading and running the harness's own unit tests in an isolated sandbox is allowed; anything that touches real infrastructure is not.\n\nDELIVERABLE: a findings list where each item has severity (blocking / non-blocking), exact path:line at the resolved head SHA, why it is exploitable or wrong, and an executable repro command or test snippet. Explicitly state which of the five scope areas you could NOT fully verify and why. Do not soften findings to reach a positive result.", "expected_role": "implementation", "id": "t_0908bc73", "title": "Audit harness security: command construction, redaction, credential exposure"} +{"board": "titan-iac", "body": "Independent read-only xhigh audit proving that the default mode of the full-handoff acceptance harness (from parent task t_6496c271) performs zero mutations.\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically (find the open/draft PR introducing the harness; do not assume a PR number). Record them and pin all findings to the resolved head SHA.\n\nSCOPE:\n1. Static command inventory: build an exhaustive inventory of every external command, API call, and SDK method the harness can reach in default mode. Walk call graphs from each entrypoint; include indirect paths through helpers, retries, and cleanup handlers. Classify each as read-only or mutating, citing path:line. A single reachable mutating call in default mode is a blocking finding.\n2. Dynamic proof via spies: run the harness in default mode with subprocess/HTTP/SDK layers replaced by spies that record invocations and hard-fail on any mutating verb (create/apply/patch/delete/rollout/scale/merge/push/comment/annotate/label, and POST/PUT/PATCH/DELETE to state-changing endpoints). Assert the recorded invocation set matches the static inventory — flag both extra calls and inventory entries never exercised.\n3. Mandatory NOT_RUN behavior: verify that any check which cannot be executed is reported as NOT_RUN and that NOT_RUN is never silently coerced into PASS or omitted from the artifact. Confirm whether the presence of any NOT_RUN forces a non-PASS overall result, and state precisely what the code actually does versus what the harness documentation claims.\n4. Escape hatches: identify every flag, env var, or config that can promote the harness out of read-only mode. Verify each requires explicit opt-in, is off by default, and is not reachable through defaulting/coercion bugs.\n\nCONSTRAINTS: worktree stays pristine — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Run only in an isolated sandbox with spied/faked I/O; never point the harness at real infrastructure.\n\nDELIVERABLE: the full command inventory table, the spy transcript, and a blocking/non-blocking findings list with exact path:line and executable repro commands. State clearly whether 'default mode is read-only' is PROVEN, PARTIALLY PROVEN, or DISPROVEN, and what would be needed to close any gap.", "expected_role": "implementation", "id": "t_0d1d6c47", "title": "Prove default mode is read-only via spies and static command inventory"} +{"board": "titan-iac", "body": "Perform a consequential independent release review of private atlas/titan-iac PR #13 at exact head 750dfa241f817179f6c0459a24651fd830e18c28 against exact base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. The supplied workspace is a clean detached worktree at the exact head. Read-only review only: do not edit files, commit, push, comment, approve, merge, deploy, mutate Kanban other than completing this assigned card, create pods, build or publish images, or expose credentials. Verify local HEAD and all available remote PR identities before trusting the diff; fail closed if identity cannot be established. Inspect the entire six-file PR, not just the last commit. Independently reproduce the prior P1 threat models: unconditional-true created-event predicate, producer hardcoded blocked status, every non-park create mislabeled blocked, late producer reassignment after the pinned ladder, and anchor drift. Confirm the real API non-sticky oracle proves ordinary create and claim, failure limit 1 blocks, recomputation at limit 2 promotes ready, while explicit initial block/unblock, dependency promotion, and current-limit circuit breaker cases remain valid. Review malformed payload handling and textual-anchor limitations for false positives and false negatives. Verify the durable placement correction excludes titan-04 and titan-19 and changes only the main Hermes CPU request 350m to 300m. Run focused regression tests, relevant Hermes unit suites in sanitized and ambient/base-comparison modes, changed-file Ruff and static/secret/diff-scope checks, kustomize build, render, and client dry-run. No image build is required because the safe unprivileged builder is a separate capability. Return a prioritized findings-first verdict: SHIP only if there are no material defects and every explicit acceptance criterion has independent evidence; otherwise BLOCK with exact file and line evidence plus reproducible input. Clearly distinguish pre-existing base failures. Human review remains required after SHIP.", "expected_role": "review", "id": "t_12a22c7f", "title": "Independent SHIP review Hermes PR 13 at 750dfa24"} +{"board": "titan-iac", "body": "Repair the live Atlas AI usage exporter reliability defect in private atlas/titan-iac, starting from exact main head 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Work in the isolated linked worktree Hermes creates from /opt/data/workspace/projects/titan-iac. Create a focused branch, commit, push, and open a pull request for human review. Do not merge or deploy/reconcile an unreviewed change. Preserve unrelated files and never expose Codex/Claude credentials, OAuth contents, tokens, or provider response bodies in Git, logs, tests, PR text, or metrics.\n\nLive reproduced evidence to verify independently:\n- services/hermes/scripts/ai_usage_exporter.py query_codex() can reach the second process.wait(timeout=5) after kill and raise uncaught subprocess.TimeoutExpired.\n- Collector.refresh_provider does not catch that exception type, so the daemon polling thread terminates.\n- /healthz continues returning 200 because it does not represent poller liveness/freshness; the ai-usage-exporter container stays Ready while quota metrics become permanently stale.\n- A metrics client disconnect also emits a noisy BrokenPipe traceback.\n\nRequired behavior:\n1. Add RED regressions first for a Codex subprocess that cannot be reaped within both bounded waits. Cleanup must be strictly bounded and must never let TimeoutExpired escape in a way that kills polling; close selector/pipes and prevent child/process leakage as far as the host API permits.\n2. Isolate every provider refresh so any unexpected provider/cleanup exception records a sanitized failure and the polling loop continues to the other provider and future cycles. Never log exception messages or response bodies because they may contain sensitive material.\n3. Make health report the polling engine, not provider quota success: allow a bounded startup grace, remain healthy when an individual provider is unavailable (that is represented by fetch_success/staleness metrics), but return non-200 when the poller thread is dead or has stopped making bounded progress. Avoid restart loops caused merely by upstream quota/API failure.\n4. Handle metrics-client disconnects without server traceback noise.\n5. Preserve last-good quota values on failures and all existing low-cardinality metric contracts. Do not scrape interactive `/status`, enable API billing, add paid provider keys, or emit account identity/credentials.\n6. Ensure the ConfigMap/script change causes the agent exporter process to restart after a reviewed Flux rollout (use the existing config-revision convention or another tracked mechanism).\n\nValidation:\n- `pytest -q testing/tests/test_hermes_ai_usage_exporter.py testing/tests/test_atlas_ai_dashboard.py` plus relevant Hermes manifest/quality tests.\n- `ruff check` for changed Python when available.\n- regenerate Atlas dashboards only if the generator changes, and prove no unintended generated drift.\n- `kustomize build services/hermes`, `kustomize build services/monitoring`, client dry-runs for both, `git diff --check`.\n- Reinspect current live logs/readiness read-only to confirm the failure signature; do not mutate live workloads before review.\n\nReport RED-before/GREEN-after evidence, exact commit/branch/PR URL, changed files, validation, explicit skips, remaining risk, and why no credential can leak. Keep the PR open, non-draft unless Gitea requires otherwise, and unmerged.", "expected_role": "implementation", "id": "t_1e95ea6d", "title": "Repair AI usage exporter poller health and cleanup"} +{"board": "titan-iac", "body": "Read the current private atlas/titan-iac main-head Hermes image patches, Dockerfile patch mechanism, and existing Kanban/Hermes scheduler tests without modifying tracked files. Trace create_task(initial_status='blocked'), event persistence, recompute_ready(), _has_sticky_block(), dependency completion handling, block_task/unblock_task, and circuit-breaker behavior. Identify the smallest patch location and exact focused tests needed to demonstrate RED-before/GREEN-after for: explicitly initially blocked tasks with no parents and with complete parents; ordinary dependency-blocked task promotion after dependencies complete; block_task/unblock_task; and sticky circuit-breaker blocks. Preserve dependency-driven promotion and official public behavior where possible. Return concrete file paths, relevant code paths, proposed assertions, and compatibility risks for the implementation worker.", "expected_role": "implementation", "id": "t_1fc75549", "title": "Analyze Hermes blocked-task state machine and regression strategy"} +{"board": "titan-iac", "body": "Amend private atlas/titan-iac PR #13 after independent BLOCK review t_06771c95. Work only in /opt/data/workspace/projects/titan-iac/.worktrees/t_cca008de on branch wt/t_cca008de, starting from exact head 4f8dcfbbf748e88726912bef1ae32201e1e640a6 and base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Verify local/remote/pull-ref/API identity first. Preserve the complete existing six-file change. Do not deploy, merge, create pods, build/publish images, run privileged containers, expose credentials, or change unrelated files.\n\nFix the single P1 review finding with adversarial proof. Dockerfile.hermes-agent patches the sticky-block consumer but does not fail closed if upstream create_task stops emitting the expected created event payload {\"status\": task_status}. Add a unique fail-closed source anchor/precondition for the producer semantics on which the consumer patch relies; do not rewrite the producer unnecessarily. dockerfiles/hermes-kanban-blocked-regression.py needs a real-API non-sticky oracle: create/claim a normal task, record one spawn failure at failure_limit=1 so it is blocked, then recompute at failure_limit=2 and prove it promotes to ready. The test must fail under either an unconditional-true created-event predicate or producer drift that labels every created event blocked, while existing explicit initial block/unblock, dependency promotion, and circuit-breaker-at-current-limit cases pass. Add malformed/non-blocked created-payload controls if needed.\n\nReproduce RED against both the over-broad predicate mutant and the exact producer-drift mutant. Prove the amended Dockerfile patch rejects producer drift before build-time tests, all anchor drift remains fail-closed, and the exact patched temp tree passes. Run focused Hermes suites in a sanitized environment (baseline 107), full isolated unit surface (baseline 357), Ruff/AST/YAML/secret/diff/line checks, kustomize render, and client dry-run. Current live service-link/health variables caused four ambient failures that reproduced on base; report both evidence sets. Do not attempt an image build without the separate safe unprivileged builder capability.\n\nCommit and push the focused amendment to wt/t_cca008de so PR #13 advances. Verify remote branch and refs/pull/13/head equal the new commit and the PR remains open/unmerged/undeployed. Status completed only after commit/push/ref verification; use incomplete while anything remains.", "expected_role": "implementation", "id": "t_20f2fd16", "title": "Repair Hermes PR 13 producer-drift regression coverage"} +{"board": "titan-iac", "body": "Perform a fresh independent read-only release review of the implementation produced by parent task t_64e1ae8d. Dynamically resolve its exact PR/head/base. Do not edit/comment/commit/push/merge/build/publish/deploy/reconcile/restart or mutate Kanban beyond returning the review.\\n\\nIndependently reproduce the zombie creation path and verify the repair prevents unreaped adopted descendants under normal completion, timeout/kill, cancellation, nested Git/SSH, concurrency, and restart without racing provider subprocess waiters or changing exit results. Audit PID1/init/subreaper/signal semantics, resource bounds and sanitized metrics. Verify compatibility with merged PR13 and then-current PR15, full quality/coverage/LOC, render/schema/client/server dry-runs, and secret scans. Return strict SHIP or BLOCK with exact executable evidence and residual rollout canary requirements.", "expected_role": "review", "id": "t_261e208f", "title": "Independent review of cli-lane process reaping"} +{"board": "titan-iac", "body": "Independently review private atlas/titan-iac PR #12 at exact head 465d386da5237f011cb408f1a3d4b7d0c6d8ffae against exact base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. This is a consequential read-only release review of the AI usage exporter reliability repair produced by parent task t_1e95ea6d.\n\nDo not edit files, mutate Kanban beyond returning this task result, commit, push, comment, merge, deploy, reconcile, or expose credentials/provider bodies. Verify local HEAD, canonical remote branch wt/t_1e95ea6d, and PR head are byte-identical before review. Inspect the entire three-file diff, not just the parent summary.\n\nReview especially: strictly bounded terminate/kill/reap behavior when both waits time out; selector/pipe cleanup and child leakage limits; per-provider and per-cycle exception isolation; last-good metric preservation; health semantics during startup, provider failure, dead poller, stalled provider, and interval sleep; avoidance of restart loops; client disconnect handling; thread safety; low-cardinality metric compatibility; sanitized logging; absence of OAuth/API-key/account data leakage; config-revision rollout behavior; test truthfulness and missing adversarial tests. Re-run the focused exporter/dashboard tests and proportionate manifest/render/dry-run validation independently. Treat parent tests and done status as evidence to verify, not truth.\n\nReturn prioritized findings with exact file/line references and reproducible inputs. State explicitly if no defect is found in an area, list exact tests/skips, and give a ship/not-ship verdict. Keep the PR open for human review.", "expected_role": "review", "id": "t_326d55c6", "title": "Independent release review of titan-iac PR 12"} +{"board": "titan-iac", "body": "Implement and validate a durable least-privilege image build/publish path for Hermes agent changes. Use the existing Jenkins/Kaniko pattern or another reviewed rootless builder. Never run privileged Docker-in-Docker, never expose unauthenticated Docker or BuildKit TCP sockets, never schedule disposable builders on titan-14 or titan-18, and never print or commit registry credentials. Deliver through a focused human-reviewed PR with tests and proof that the image digest can be published and then updated through Flux. PR #13 is intentionally undeployed until this capability exists.", "expected_role": "implementation", "id": "t_404f607c", "title": "Provide a tracked unprivileged Hermes image build and publish workflow"} +{"board": "titan-iac", "body": "After t_c31dd427 completes, independently review exact repaired PR18 head against current main. Re-run the complete t_dbdcd739 and t_5975c06a evidence, especially lease-failure persistence/restart recovery, exact-run fencing, no poison-row CrashLoop, mediator/worker drain and RWO recovery, retry branch submission, startup tooling/Flux health, focused ancestry not bundling other open drafts, all security boundaries, three-node scheduling, at least 95 percent line and branch coverage, below 500 LOC, full quality/render/dry-run/secret scans. Read-only. Return completed SHIP or completed BLOCK; a BLOCK verdict completes the review. No edits/comments/push/merge/deploy/reconcile.", "expected_role": "review", "id": "t_529c50f9", "title": "Independent re-review of final PR 18 pool repair"} +{"board": "titan-iac", "body": "Perform a new independent read-only consequential review after parent t_8fbeb41f repairs existing PR #18. Dynamically resolve exact PR/head/base and current PR14/15/16/19 heads; distrust all claimed tests. No edits/comments/commit/push/merge/build/publish/deploy/reconcile/live mutation.\\n\\nRe-run every artifact under /opt/data/workspace/evidence/t_dbdcd739 and independently test all P0/P1/P2 integration boundaries: run ID typing/finalization/reconcile progress; workspace/branch/project routing across all Atlas boards; PR15 module/runtime compatibility; per-worker crypto isolation; sidecar/model network trust; PR14 broker-only SCM; OAuth refresh ownership; exact lease expiry/reassignment/error handling; worker-only three-node schedulability; config rollout/version skew; RBAC/network/mount isolation; canonical coverage/LOC/full gate/renders/schema/dry-runs/secret scans. Return strict SHIP or BLOCK as a completed review; findings do not require you to edit the reviewed code.", "expected_role": "review", "id": "t_5975c06a", "title": "Independent re-review of repaired PR 18 distributed pool"} +{"board": "titan-iac", "body": "After parent repair t_69159fea completes, perform a fresh independent read-only xhigh review of draft PR #19 at its exact then-current head against then-current main. Do not trust previous claims or the earlier review. Keep the review worktree pristine and do not edit, comment, commit, push, merge, publish, deploy, reconcile, create credentials, or mutate live resources.\n\nRe-run every adversarial class from review t_e7f46d44/Codex session 01a00f52-a815-7933-9f7d-04ac16145027: structural command allowlist, credential paths, raw Secret/kubeconfig/Helm/Git controls, subprocess memory/time/process-group bounds, redact-before-truncate across JSON and human output, unknown/NaN/future/malformed/stale evidence, PATH/binary provenance, armed preflight/create uncertainty/cleanup, remote-main+PR+image+running-revision binding, three-node and chat/Telegram coverage, dependency/rollback accuracy, and zero/partial/malformed states. Prove default mode performs no mutation through spies and a complete command inventory.\n\nRequire every production module in the canonical managed/lint/line+branch coverage contract at >=95 percent and every source/test file below 500 LOC. Require full canonical quality gate, Ruff/format/compile, mutation/adversarial suite, render/dry-run, diff and secret scans all green. Return a strict completed SHIP or completed BLOCK review with exact executable evidence. A BLOCK verdict completes this review task; do not attempt implementation work.", "expected_role": "review", "id": "t_5ae177fc", "title": "Independent re-review of repaired PR 19 acceptance harness"} +{"board": "titan-iac", "body": "Work in the isolated Titan IaC worktree created for this task. Read AGENTS.md and HERMES_ATLAS_MIGRATION_HANDOFF.md completely. Use Claude Code through Claude.ai subscription OAuth only; never configure/use ANTHROPIC_API_KEY or any Console/API-billed route.\n\nBuild the hardest remaining independent artifact: a fail-closed, read-only-by-default Hermes full-handoff acceptance harness and release runbook. Keep the change isolated to new files scripts/ops/hermes_handoff_acceptance.py, testing/tests/test_hermes_handoff_acceptance.py (split into meaningful modules if it would exceed 500 LOC), and docs/hermes_full_handoff_acceptance.md. Do not edit files changed by open PRs #14, #15, #16, #17, or the active distributed-worker task/PR. PR #13 is already merged/deployed; baseline origin/main must be ab346f55509d584e457fe26cf90be3078f7a375c or a descendant. Dynamically fetch and record exact current heads/state for #14-#17 and any distributed-worker PR; do not hard-code stale heads.\n\nThe harness must emit machine-readable JSON plus a concise human summary, redact credential-shaped values, never print environment values/tokens/private keys/cookies/Vault responses, enforce bounded timeouts/output, and classify every check PASS/FAIL/NOT_RUN/NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN makes overall NO_GO. Split evidence into an external read-only operator vantage point and an actual in-pod Hermes self-probe; do not treat kubectl auth can-i --as alone as proof. Default mode must perform no mutation. Feature-branch push and draft-PR creation must be a separately armed ephemeral mode with exact unique refs, explicit confirmation, preflight, protected-main/master rejection, and verified cleanup; default reports these NOT_RUN.\n\nVerify after pending PRs merge/deploy: distinct chat/agent/triage scopes; Codex ChatGPT login and Claude claude.ai firstParty subscription with no provider API-key env vars; Switchyard provider/model/effort/fallback/latency/failure evidence; visible durable session/Kanban activity; denied cluster-admin/secrets/SA-token/impersonation/arbitrary exec-attach-portforward/workload mutation; allowed required get/list/watch/pod logs/Flux+Helm status; no worker Gitea credential and no merge/approve/close/admin route; broker clone/fetch/feature push/metadata/draft PR and protected-branch denial; every node dedicated locked hermes-agent account with no human key/sudo/disk/runtime/K3s/kubelet access; tokenless unprivileged builder and exact expected caps; immutable Harbor build tag; finalization/replay/decomposition regressions; no unexpected GitOps suspension/unhealthy state; three distinct distributed workers with coordinator-only state ownership, distinct nodes/PVCs, authenticated bounded activity/result path, stale/duplicate safety; activity visible in agent UI; chat.hermes Telegram source/topic continuity and no blank session render regression.\n\nDocument exact dependency/merge order for merged #13 plus pending #14-#17/distributed pool, image build/Flux rollout, rollback points, and final go/no-go checklist. Unit-test parsing, redaction, timeout, partial failure, zero-state, dual-vantage mismatch, mandatory skip, and ephemeral arming/cleanup behavior. Keep production files <500 LOC and achieve >=95% per-file coverage for new source. Run focused/full relevant tests, Ruff/format, py_compile, diff/secret scans and applicable render checks. Commit, push a feature/hermes-full-handoff-acceptance branch, create a draft PR only, report exact SHA/evidence. Do not merge, publish images, deploy, reconcile, create credentials, or mutate live cluster/Gitea during this implementation.", "expected_role": "implementation", "id": "t_6496c271", "title": "Build fail-closed Hermes full-handoff acceptance harness"} +{"board": "titan-iac", "body": "Live reliability defect observed on the merged PR13 agent.hermes cli-lane-runner pod: after ~95 minutes it has 141 defunct git children, all parented to PID 1 (/opt/coordinator/cli_lane_runner.py). This is an authorized repair of our own runtime.\\n\\nDiagnose and repair subprocess lifecycle/reaping so long-lived agent.hermes workers cannot accumulate zombies or exhaust PID capacity while running Codex/Claude/Git/test commands. Work on current atlas/titan-iac main in an isolated worktree and draft PR. Do not manually mutate/deploy/restart production.\\n\\nRequirements:\\n- Reproduce the actual creation path and identify which spawn/session/process-group behavior leaves children for PID1.\\n- Implement deterministic reaping/cleanup at the owner boundary without stealing children from active provider subprocesses, corrupting streamed output, or masking exit status.\\n- Cover normal exit, timeout SIGTERM/SIGKILL, provider cancellation, nested Git/SSH helpers, concurrent lanes, runner restart, and orphan adoption. Avoid a broad signal handler if it races subprocess waiters; use an explicit supervisor/subreaper/init boundary as appropriate and justify it.\\n- Add bounded observability for active/reaped/orphaned child counts without command lines, environment, credentials, or sensitive output.\\n- Preserve PR13 blocked-task behavior and PR15 finalization/recovery compatibility. Resolve current exact PR15 head dynamically and prove clean semantic merge in the reachable order.\\n- Keep every production/test file under 500 lines, >=95% per-file coverage, quality gate green; render/dry-run Hermes and run relevant static/secret checks.\\n- Commit/push/open one draft PR, human review required. No image publication, merge, Flux reconciliation, deployment, live restart, or credential access.\\nReturn completed only with exact pushed SHA and evidence.", "expected_role": "implementation", "id": "t_64e1ae8d", "title": "Eliminate cli-lane zombie process accumulation"} +{"board": "titan-iac", "body": "Repair existing draft PR #19 on its SAME branch and implementation worktree after independent review t_e7f46d44 returned BLOCK twice. Exact reviewed head 8f005458282269ba5c07941814e4237f5d4cf3ac; resolve current main and all dependent PR heads dynamically. Do not create another PR.\n\nTreat the completed review transcript in Codex session 01a00f52-a815-7933-9f7d-04ac16145027 as authoritative evidence. Close every release blocker:\n1. Default mode must be structurally read-only and credential-safe. Reject kubectl auth reconcile, config view --raw, Secret/raw-secret reads, client-side or post-separator dry-run tricks, git fetch/config sshCommand, Helm values, and shell paths under service-account/Vault/runtime credential roots before subprocess creation.\n2. Bound stdout/stderr in memory while the child runs; enforce absolute timeout, kill/reap the whole process group, bound descendants/pipes, validate timeout/output/concurrency inputs, and make truncation fail closed for every evaluator.\n3. Redact before truncation and screen every JSON plus human summary/error/identity field. Never expose a credential prefix.\n4. Fail closed on unknown statuses, empty catalog, NaN/non-finite numbers, future timestamps, malformed Flux/routing rows, stale required-provider evidence, generic 404/no-route strings, or mocked/self-authored PATH evidence. Add binary/path attestation or equivalent provenance.\n5. Armed mode must preflight before any networked/mutating vantage, bind actual repo to fixed expected Atlas repo, reserve cleanup time, verify exact draft/base/head state, discover uncertain creates, and require branch plus PR cleanup success. Never false-PASS malformed/failed cleanup.\n6. Freshness must bind remote main SHA, reviewed PR head, image digest/imageID, build SHA, and running deployment revision. Require all three distributed workers/nodes; no vacuous node-count=1 default. Include chat/Telegram continuity and pool assignment checks.\n7. Add all 15 production modules to canonical managed_modules, lint_paths, and coverage.tracked_files. Every new/modified source must remain below 500 LOC and at least 95 percent line and branch coverage. Fix the two environment-sensitive tests by isolating provider-health and Gitea-token fixtures. Canonical quality gate, Ruff, format, compile, mutation/adversarial tests, render/dry-run, diff/secret scans must all be green.\n8. Correct dependency/merge/conflict graph using exact current heads. Remove unsafe rollback guidance restoring cluster-admin and false PVC-pruning assumptions.\n9. Add executable regressions for every reviewer repro, including large/malformed inputs, zero state, partial failures, timeouts, descendant pipes, and no-subprocess/no-mutation spies.\n\nKeep the worktree pristine except this repair. Amend/commit and force-with-lease only the existing feature/hermes-full-handoff-acceptance branch, update PR #19, report exact head and evidence. Human review required. No merge, publish, deploy, reconcile, credential access, or live mutation.", "expected_role": "implementation", "id": "t_69159fea", "title": "Repair PR 19 full-handoff acceptance blockers"} +{"board": "titan-iac", "body": "After t_888cdeee completes, independently review exact repaired PR19 head against current main. Re-run all prior policy, subprocess, redaction, evaluator, empty-evidence, attestation, armed cleanup, freshness, three-node/chat, dependency, mutation, coverage and format repros. Require mandatory names-absent checks to return NOT_RUN on zero evidence; exact repo/path/impersonation boundaries; at least 95 percent line and branch coverage; below 500 LOC; full canonical green gate. Read-only. Return completed SHIP or completed BLOCK; a BLOCK verdict completes the review. No edits/comments/push/merge/deploy/reconcile.", "expected_role": "review", "id": "t_699f9a2d", "title": "Independent re-review of final PR 19 harness repair"} +{"board": "titan-iac", "body": "Live failure observed on t_dbdcd739: a read-only independent release reviewer returned a completed BLOCK review with five P0 findings, but the local goal judge rejected completion and resumed Claude with an instruction to continue because the reviewed implementation was not SHIP. A review's job is to issue a verdict, not repair the implementation; this wastes subscription capacity and can loop.\\n\\nRepair Hermes goal/result semantics so task-role completion is judged against the assigned action. For review/diagnostic/audit tasks, status=completed with a truthful SHIP or BLOCK verdict and findings must finalize even when findings show the reviewed artifact is unfit to ship. Only genuine inability to perform the review belongs in blockers. Implementation tasks must still be held to their acceptance criteria.\\n\\nRequirements:\\n- Reproduce with exact task body/result shape and local judge path.\\n- Define a deterministic task-role/expected-output contract, preferably explicit metadata/schema rather than fragile keyword-only inference; preserve backward compatibility.\\n- Make reviewer/diagnostic completion fail closed on malformed/missing verdict/evidence, but never demand code edits prohibited by read-only scope.\\n- Preserve implementation goal looping and PR15 accepted-result durability/exact-run behavior. Dynamically merge-test current PR15 head.\\n- Cover SHIP, BLOCK-with-findings, incomplete review, blocked review, implementation incomplete, ambiguous tasks, reviewer provider fallback, restart/replay, and no duplicate turn/session.\\n- Emit bounded/sanitized judge reason and correct Kanban final result; no hidden auto-mutation.\\n- All files <500, >=95% per-file canonical coverage, quality/render/dry-run/secret checks.\\n- Draft PR only, human review; no merge/build/publish/deploy/reconcile/live restart.\\nReturn exact pushed SHA and evidence.", "expected_role": "implementation", "id": "t_6da029e0", "title": "Make goal judge finalize completed review verdicts"} +{"board": "titan-iac", "body": "Fresh independent read-only xhigh release review of chat.hermes smoothness work from parent t_83252d81. Resolve exact draft PR/head/base; do not edit/comment/push/merge/deploy/reconcile/call real Telegram or mutate session data. Reproduce blank/inaccessible session rendering, reconnect/resume, active tool progress, API/Telegram source tags and legacy migration, deterministic durable topics across hour/day/week/router restart, bounded recent-turn+summary context and duplicate suppression, malformed/oversized history, four-tenant isolation, inbound/outbound media and path safety, and truthful previous-image edit semantics. Audit Switchyard/manual route preservation and chat/agent/triage separation. Run focused/full UI/session/router Go+race/Python tests, coverage/LOC, Kustomize/schema/client+Flux-manager server dry-runs, secret/diff scans. Return strict SHIP/BLOCK with exact executable evidence and residual risks. Keep review worktree pristine.", "expected_role": "review", "id": "t_7b7eaaf5", "title": "Independent review of chat.hermes smoothness"} +{"board": "titan-iac", "body": "Starting from the current private atlas/titan-iac main head, create an isolated linked worktree and dedicated branch. Use the two analysis reports to implement the smallest robust Hermes-only change. First run or add focused tests that fail against the current behavior, retaining RED evidence; then patch the existing Dockerfile image-patching/test mechanism so explicitly created blocked tasks acquire sticky blocked semantics without weakening normal dependency promotion, explicit unblock behavior, or circuit-breaker blocks. Add focused GREEN tests for every required scenario. Update deployment configuration to set HERMES_CLI_LANE_CONCURRENCY=2, cap cli-lane-runner CPU limit at 2, and set enableServiceLinks: false at the hermes-agent pod-spec level; add a manifest regression proving service links are disabled and check resource requests remain sensible. Preserve unrelated work and never expose or commit credentials. Do not deploy, reconcile Flux, restart the agent, or mutate live Kanban data beyond this task lifecycle. Record exact changed files and RED/GREEN commands/results for handoff.", "expected_role": "implementation", "id": "t_7ddd9eb2", "title": "Implement Hermes isolation and blocked-task regressions on a branch"} +{"board": "titan-iac", "body": "Make chat.hermes.bstein.dev smoother and truthful without changing agent.hermes or triage.hermes scope. Work in an isolated worktree and read AGENTS.md, HERMES_ATLAS_MIGRATION_HANDOFF.md, current chat/router/session code, and exact current open PR #14-#17 plus distributed-pool diff before editing. Avoid every file changed by those PRs; if a required fix overlaps, stop with a precise dependency instead of racing it. Use subscription Codex only; no provider API keys.\n\nFirst perform a read-only live/code audit, then implement only evidence-backed fixes in a focused draft PR: eliminate blank/inaccessible session render states and ensure activity progressively renders during long work; preserve reconnect/resume and lineage; label existing and new Telegram API sessions under Telegram rather than Unassigned; keep durable topic-based Telegram continuity for roughly week-scale conversations with bounded recent turns + durable compact summary rather than resending an unbounded transcript; make source/topic/session identity deterministic across pauses and router restarts; prevent duplicate stored context; keep text/image/tool capabilities truthful; preserve actual outbound Telegram media delivery and never expose MEDIA paths; explicitly distinguish whether an image-edit request can reuse a prior image attachment versus generate a new image; do not claim inbound image editing if transport does not provide the attachment.\n\nAdd adversarial tests for existing-session migration idempotence, source labels, topic expiry/reuse, summary/recent-turn bounds, duplicate suppression, resume after an hour/day/week boundary, router restart, malformed/oversized histories, WebSocket/poll disconnect, blank render fallback, active tool progress, and Telegram media/caption safety. Preserve four-tenant isolation and Switchyard routes/manual overrides. No secrets in Git/logs. Keep production files <500 LOC and >=95% per-file coverage for new source. Run relevant full tests, Ruff/Go tests/race where applicable, Kustomize/client/Flux-manager server dry-runs, diff and secret scans. Commit/push branch feature/hermes-chat-smoothness, open draft PR for Brad. Do not merge, publish, deploy, reconcile, call real Telegram, or mutate live session data.", "expected_role": "implementation", "id": "t_83252d81", "title": "Harden chat.hermes continuity and activity rendering"} +{"board": "titan-iac", "body": "Repair existing Titan IaC draft PR #12 on its SAME branch/worktree; do not open a replacement PR. Start by read-only fetching current origin/main ab346f55 or descendant and exact current PR14 head because PR14 is splitting baseline oversized tests. Rebase/merge safely without dropping PR13/PR14 semantics; if PR14 is not merged, validate a synthetic PR12+PR14 composition and document required merge order. The exporter reliability change remains unique and must not be closed.\n\nMandatory repair: services/hermes/scripts/ai_usage_exporter.py is now 612 LOC and violates the repository 500-LOC source rule even though the legacy quality gate does not track that script. Extract meaningful bounded Codex process/query cleanup and/or polling/HTTP engine modules so every changed production file is <500 LOC; do not add exclusions or meaningless chunks. Preserve structured subscription-only quota collection, no API keys, sanitized class-name-only errors, bounded process reaping, provider isolation, last-good samples, poller-progress health, and disconnect handling. Add/retain >=95% per-file coverage and adversarial cleanup/thread/dead-health tests. Rebase current main, update config revision without clobbering newer rollout annotations, run exporter/dashboard tests, relevant full Hermes tests, Ruff/format/compile, actual make test/quality after composing PR14 as required, kustomize Hermes+monitoring, client and Flux-manager server dry-runs, diff/secret scans. Amend/force-with-lease only existing PR12 branch, verify PR #12 stays open/unmerged with exact head, stop for fresh independent review. No merge/build/publish/deploy/reconcile/live mutation.", "expected_role": "implementation", "id": "t_87cd9076", "title": "Rebase and split AI usage exporter PR 12"} +{"board": "titan-iac", "body": "Repair existing draft PR #19 on its SAME feature/hermes-full-handoff-acceptance branch/worktree after independent review t_5ae177fc BLOCKED exact head c808baff40a12a90f43c2e5e39b94d139f88ea29. Do not create another PR.\n\nFix the mandatory P1 exactly: evaluate_names_absent must never PASS on rc=0 with zero observations. Return NOT_RUN/NO_GO on empty evidence and on pipeline/upstream/tool/catalog/jsonpath drift. Add all five real catalog regressions plus shell-pipeline and kubectl jsonpath empty-output repros.\n\nAlso close the review's reachable hardening/evidence defects: exact repo path boundary, reject dot segments, structurally forbid impersonation in all vantages unless a narrowly explicit audited inner self-probe requires it, correct the runbook format command so it passes, remove or implement inert concurrency/Telegram flags honestly, remove unattestable unused flux/helm entries or pin them, paginate/discover uncertain draft creation and surface/manual-clean exact residue, retain executable provenance for record-false steps, and do not add broad legacy LOC exceptions in this PR; coordinate with repaired PR14/15 canonical contract instead.\n\nPreserve all previously verified read-only/default safety, bounded subprocess/redaction, fail-closed parsing, binary attestation, armed cleanup, freshness/lineage, three-node/chat checks, at least 95 percent line and branch coverage and below 500 LOC. Re-run full canonical quality and every earlier adversarial/mutation test, Ruff format/check, compile, kustomize/dry-runs, exact current dependency graph, diff/secret scans. Amend/push only existing PR19 branch and stop for fresh independent review. No merge/publish/deploy/reconcile/live mutation/credential access.", "expected_role": "implementation", "id": "t_888cdeee", "title": "Repair PR 19 zero-evidence fail-open"} +{"board": "titan-iac", "body": "Independent read-only xhigh correctness audit of the result/verdict machinery of the full-handoff acceptance harness (from parent task t_6496c271).\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically; do not trust any claimed coverage or claimed go/no-go behavior. Pin findings to the resolved head SHA.\n\nSCOPE:\n1. JSON artifact schema: verify the emitted artifact conforms to a declared schema, that the schema is enforced (not merely documented), and that required fields cannot be absent. Check schema versioning and behavior on unknown/extra fields.\n2. Deterministic overall verdict: derive the truth table mapping per-check states (PASS / NO_GO / NOT_RUN / error) to the overall result. Verify it is total (no unhandled combination), order-independent, and free of nondeterminism from dict/set iteration, timestamps, concurrency interleaving, or locale. Prove NO_GO cannot be masked by a later PASS and that the fail-closed default is NO_GO, not PASS.\n3. Bounded execution: verify enforced upper bounds on wall time per check and overall, on captured output size (stdout/stderr truncation without losing the verdict), and on concurrency (bounded worker pool, no unbounded fan-out). Check timeout behavior — a timed-out check must become NOT_RUN or NO_GO, never PASS.\n4. Degenerate inputs: exercise partial evidence, malformed/truncated JSON, and zero-state (no pods, no revisions, empty pool). Each must yield an explicit non-PASS state rather than a crash or a vacuous PASS. Vacuous PASS on zero-state is a blocking finding.\n5. Evidence integrity: verify dual operator-side and in-pod evidence are independently collected and cross-checked, that revision/image freshness is validated against the resolved head SHA rather than accepted from a label or claim, and that the harness cannot be made to pass on mocked or self-authored evidence. Test this concretely: craft a forged evidence bundle and confirm the harness rejects it.\n\nCONSTRAINTS: worktree pristine — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Use fixtures and fakes only.\n\nDELIVERABLE: the verdict truth table you derived from the code, the degenerate-input and forged-evidence experiment results, and a blocking/non-blocking findings list with exact path:line and executable repros. Call out every place implemented behavior diverges from the documented contract.", "expected_role": "implementation", "id": "t_8ceae30f", "title": "Verify determinism, JSON schema, and evidence integrity of harness verdicts"} +{"board": "titan-iac", "body": "Independent read-only xhigh review of the dependency, merge, rollout, and rollback plan for the full-handoff acceptance harness (from parent task t_6496c271), evaluated against actual current state rather than the plan's own claims.\n\nSETUP: resolve dynamically — the harness draft PR (number, head SHA, base SHA), and the real current status of each of PR12 through PR17 (open/draft/merged/closed, base branch, head SHA, merge conflicts, CI status). Also resolve the current distributed-pool state the rollout targets. Record what you observed with timestamps; do not rely on any status asserted in the PR description or prior task notes.\n\nSCOPE:\n1. Dependency correctness: verify the stated ordering among PR12-PR17 and the harness PR matches the actual code/config dependencies. Identify any dependency claimed but not real, or real but unclaimed. Flag any cycle or any PR that would break if merged in the stated order.\n2. Merge plan: check each PR's base branch and rebase state, detect textual and semantic conflicts between the harness PR and the others (especially shared config, schema, and entrypoint files), and assess whether the stated merge sequence is executable as written today.\n3. Rollout plan: evaluate staging against the current distributed-pool state — is the plan valid for the pool's actual size, versions, and heterogeneity? Check gating between stages, blast radius per stage, and whether the acceptance harness is a hard gate or merely advisory.\n4. Rollback plan: verify rollback is concrete and executable (exact revisions/images to revert to), covers partial-rollout states, and does not depend on the very component being rolled back. Identify any irreversible step, data/schema migration, or one-way door, and any state a rollback would strand.\n5. Freshness: confirm the plan's referenced revisions/images still exist and are current; stale references are findings.\n\nCONSTRAINTS: strictly read-only — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Query state; do not change it.\n\nDELIVERABLE: an observed-state table for PR12-PR17 and the pool, then a blocking/non-blocking findings list with exact references (PR number, path:line, or resource identifier) and, for each blocker, the specific correction the plan needs. Explicitly state whether the plan is executable as written against today's state.", "expected_role": "implementation", "id": "t_8f526175", "title": "Review dependency, merge, rollout and rollback plan against PR12-PR17 state"} +{"board": "titan-iac", "body": "Repair existing draft PR #18 on its SAME branch/worktree after independent review t_dbdcd739 returned BLOCK. Exact reviewed head: 20002527512235b29054f3636b02418168373009; current main dynamically resolve. Review evidence is durable under /opt/data/workspace/evidence/t_dbdcd739. Do not create a second PR.\\n\\nAll five P0 integration defects must close:\\n1. Coordinator.finalize compares integer current_run_id to string binding run_id, marks every result stale, then conflicting assignments wedge reconcile. Canonicalize at one boundary and use exact-run-safe DB APIs.\\n2. Existing task.workspace_path is normal for live tasks; do not capability-block every task. Define safe distributed workspace migration/ownership without shared mutable worktrees and preserve active/untracked ownership.\\n3. Accept Hermes-established safe branch conventions including wt/ and review/... in addition to feature/fix/etc, using actual Git ref validation and traversal denial.\\n4. Task has no repo_url/base_branch fields and current design sends every board to titan-iac. Resolve repo/base from the canonical project/board registry or explicit validated task metadata persisted by a reviewed schema migration; cover every Atlas project including Metis.\\n5. Exact merge with current PR15 must include/import every split cli_lane module and preserve its finalization semantics. No text-only compatibility claim.\\n\\nAlso close consequential handoff/security/reliability findings:\\n- Do not share one HMAC authority among all workers; derive/bind per-ordinal credentials or equivalent cryptographic isolation.\\n- Require node-role worker for execution Pods; avoid control-plane. Prefer healthy accelerator/rpi5 capacity but prove 3 distinct schedulable nodes and avoid titan-22/24/unhealthy nodes without starving Switchyard/Flux.\\n- The model container must not be able to call localhost sidecar APIs to bypass commit/untracked/result gates. Add a real mediation boundary (separate pod/network identity or unforgeable per-request capability), not instruction-only trust.\\n- Integrate the reviewed PR14 hermes-scm broker; never stage raw developer-gitea token into model-facing worker pods or bypass broker policy.\\n- Make subscription OAuth refresh ownership safe: model may use writable per-worker copies, but cannot write authoritative shared Vault credential state or the pool HMAC derivation input; prevent multi-writer refresh-token lost updates.\\n- Enforce lease expiry/reassignment with exact run fencing; surface worker exceptions to Kanban and release ordinals rather than spin; config changes must trigger controlled versioned rollout/negotiation.\\n- Remove dead/asymmetric egress, narrow credential-sync mounts, preserve tokenless SA/RBAC/network isolation.\\n- Add all new production modules to canonical lint/coverage contract, every source >=95% branch/line coverage and <500 LOC; split test baselines meaningfully if current main still needs it.\\n- Fix the env-sensitive Cassandra token test only if not already closed by PR14; do not hide failures.\\n- Run independent-style executable tests from review repros, focused/full quality, exact synthetic merges with current PR14/15/16/19, kustomize/kubeconform/client+Flux-manager server dry-runs, scheduling simulation, secret/diff scans.\\n\\nCommit/amend and force-with-lease the existing feature/hermes-distributed-worker-pool branch, update PR #18, report exact head and evidence. Human review required. No merge, image publish, Flux reconcile, deployment, live worker/node/account mutation, or credential access.", "expected_role": "implementation", "id": "t_8fbeb41f", "title": "Repair PR 18 distributed pool integration blockers"} +{"board": "titan-iac", "body": "Independently execute and report the full quality gate for the full-handoff acceptance harness (from parent task t_6496c271). Do not accept any claimed numbers — regenerate every metric yourself.\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically (locate the open/draft PR introducing the harness), check out the head SHA read-only, and record the exact changed-file list the gates apply to.\n\nSCOPE:\n1. LOC limit: measure every source and test file added or modified by the PR and verify each is under 500 LOC. Report the exact count per file, name any violations, and state your counting method (physical lines) explicitly.\n2. Per-file coverage: run the test suite with per-file coverage reporting and verify every source file in the PR reaches >= 95%. Report the actual per-file percentage table and every uncovered line range for files below threshold. Aggregate/total coverage is not acceptable evidence.\n3. Mutation testing: run mutation tests over the harness modules. Report the mutation score and enumerate surviving mutants with path:line — surviving mutants in verdict-computation, redaction, or read-only-enforcement code are blocking.\n4. Adversarial tests: run the adversarial/negative test suite. If gaps exist, describe the missing cases (do not commit new tests).\n5. Static and hygiene gates: run Ruff lint and Ruff format check, byte-compile all Python sources, produce the diff against the resolved base SHA, and run a secret scan over both the diff and the full tree. Report exact command lines and exit codes for each.\n\nCONSTRAINTS: keep the worktree pristine. You may run tests and read-only tooling in a sandbox, but make no edits, no formatting fixes, no comments, no commits, pushes, merges, deploys, reconciles, or live infrastructure calls. If a gate cannot run, report it as NOT_RUN with the reason — never infer a pass.\n\nDELIVERABLE: a table of gate -> exact command -> exit code -> result (PASS / FAIL / NOT_RUN), the full per-file LOC and coverage tables, the surviving-mutant list, and raw tool output for every failure. Flag each failure as blocking or non-blocking with a one-line justification.", "expected_role": "implementation", "id": "t_a585e217", "title": "Run quality gates: LOC limits, per-file coverage, mutation and adversarial tests"} +{"board": "titan-iac", "body": "Implement the first production-safe distributed execution pool for agent.hermes in atlas/titan-iac. The purpose is to move all engineering work now done by local Codex/Claude into agent.hermes while spreading three concurrent workers across distinct cluster nodes.\n\nAuthoritative constraints:\n- Keep exactly one stateful coordinator owning Hermes Kanban/SQLite and the existing RWO hermes-agent-home. Never scale that Deployment above one and never mount its Kanban database or home PVC in a worker.\n- Add three execution workers that can be scheduled on distinct hostnames. Prefer the larger healthy ARM64 accelerator nodes, then healthy rpi5 fallback; use pod anti-affinity/topology spread and honest requests. Do not use titan-22 or titan-24. Preserve all current unhealthy-node exclusions.\n- Each worker needs its own durable RWO workspace/session volume (StatefulSet volumeClaimTemplates is acceptable) so provider sessions and Git state survive pod/node restart without cross-worker filesystem sharing.\n- The coordinator must remain the only component that claims/finalizes Kanban runs. Implement a bounded authenticated assignment/result/heartbeat channel that binds board, task_id, run_id, worker ordinal, attempt, and payload digest. Stale or replacement-run results must never complete/reclaim another run. Duplicate delivery/restart must be idempotent. Lost workers must be recoverable without double execution.\n- Workers clone/fetch only the assigned Atlas repo/feature branch through the reviewed SCM boundary; do not place a raw Gitea token in a model-facing container. Do not use a shared mutable Git worktree. Preserve unrelated/untracked files when a task explicitly owns them or fail closed with a clear blocker.\n- Use only Claude.ai subscription OAuth and ChatGPT/Codex subscription auth from Vault/runtime memory. No ANTHROPIC_API_KEY, OPENAI_API_KEY, Console/API-billed path, secret in Git/ConfigMap/log/command line. Credential refresh targets must be writable and restart-safe.\n- Give workers only the cluster/SCM capabilities required by their role. No cluster-admin, Secrets reads, service-account token creation, workload mutation, exec/attach/port-forward, or node-root access unless routed through a separately reviewed explicit operator boundary. automountServiceAccountToken must default false.\n- Preserve Switchyard automatic provider/model/effort routing and cross-provider fallback. Provider session continuity must remain attached to the exact task/run.\n- Stream sanitized worker activity, route, heartbeat, node/ordinal, and final evidence back into the existing Kanban log/session activity UI so the user can see continuous work at agent.hermes.bstein.dev.\n- Add bounded payloads, deadlines, concurrency, disk retention/GC, nofollow/atomic writes where applicable, and fail-closed startup/readiness checks.\n- Add executable adversarial tests for simultaneous claims, duplicate assignment/result, stale run, coordinator restart, worker restart, heartbeat loss, node replacement, oversized/malformed payloads, path traversal/symlinks, credential absence, cross-worker isolation, three-node scheduling contract, and visible activity.\n- Preserve compatibility with merged PR #13. Inspect open draft PRs #14/#15/#16/#17 and either avoid their surfaces or prove clean synthetic merge behavior; do not copy unreviewed assumptions from them.\n- Render services/hermes and every new Flux Kustomization, run client and Flux-equivalent server dry-runs, relevant full tests, Ruff/compile/shell/diff/secret scans. Keep source files below the repository quality limits.\n- Work only on a feature branch/worktree. Commit and push intentionally, then create one draft Gitea PR with exact validation evidence. Do not merge, publish an image, reconcile Flux, deploy, mutate live worker/node accounts, or claim live three-node proof before human review and rollout.\n\nReturn completed only when the draft PR exists at an exact pushed SHA and all implementable local acceptance checks are green. Findings belong in findings; blockers only for genuine obstacles preventing the task itself.", "expected_role": "implementation", "id": "t_af8d08d0", "title": "Implement a safe three-node agent.hermes execution pool"} +{"board": "titan-iac", "body": "Fresh independent read-only review of the implementation produced by parent t_6da029e0. Dynamically resolve exact PR/head/base. Do not edit/comment/commit/push/merge/build/deploy/reconcile.\\n\\nReproduce the original reviewer-loop incident and verify completed review/audit/diagnostic verdicts finalize whether SHIP or BLOCK-with-findings, while malformed/incomplete reviews and unfinished implementation goals continue or fail correctly. Audit explicit role metadata, backward compatibility, restart/replay, provider fallback, session/turn duplication, PR15 semantic merge, bounded sanitized judge evidence, quality/coverage/LOC/render/dry-run. Return strict SHIP/BLOCK findings with exact evidence.", "expected_role": "review", "id": "t_b04a5e67", "title": "Independent review of goal-role completion semantics"} +{"board": "titan-iac", "body": "Repair existing draft PR #22 on the SAME feature/hermes-review-goal-semantics branch/worktree after independent review t_b04a5e67 BLOCKED exact head b080b5f622ae0998213f3287762aea30dc931a73. Do not create another PR.\n\nClose every blocker: pass the resolved task role into unfinished_result_reason at both current and PR15 call sites so completed review BLOCK/SHIP cannot be short-circuited by implementation unfinished prose; parse explicit role/output directives from both real newlines and canonical literal backslash-n task bodies without allowing injection; make legacy inference correctly classify the real review/audit/diagnostic card corpus while conflicting or mutating cards fail closed; exclude all runner-appended goal rejection/controller history from card_scope; preserve and document deliberate single-shot semantics; prevent old accepted terminal journals from becoming invalid across upgrade or provide versioned compatibility; do not let changed-files/mutation evidence self-certify as review; use the canonical redactor extended for bare-hex Gitea PAT/basic-auth URL without reading real credentials; pin upstream context-heading contract. Add exact live-card/recovered t_dbdcd739/end-to-end execute_claim regressions. Preserve implementation goal judging, bounded safe reasons, PR15 compatibility, at least 95 percent line and branch coverage, below 500 LOC, full quality/render/dry-run/format/secret gates. Amend/push only existing PR22, no merge/deploy/reconcile.", "expected_role": "implementation", "id": "t_bc268843", "title": "Repair PR 22 review-role goal semantics"} +{"board": "titan-iac", "body": "Provide an authenticated noninteractive Forgejo/Gitea API wrapper and concise worker guidance for scm.bstein.dev. It must use only runtime-injected credentials from the existing Vault/credential-broker path, never print tokens, never persist credentials in Git/workspaces/caches, restrict hosts and TLS verification, and make common read operations (PR head/base/state, comments/reviews/checks, branch metadata) straightforward. Mutating operations must be explicit and preserve the human-review default; no merge endpoint by default. Add tests proving URL/host allowlisting, redaction, missing-credential failure, and read-only behavior. Deliver through a focused PR and validate from an isolated Hermes worktree.", "expected_role": "implementation", "id": "t_bf2ff6ec", "title": "Give Hermes workers a first-class safe Gitea PR API client"} +{"board": "titan-iac", "body": "Repair existing draft PR #18 on its SAME feature/hermes-distributed-worker-pool branch/worktree after independent review t_5975c06a BLOCKED exact head 689bcb6e48414854447988ba174143a307d308b9. Do not create another PR.\n\nP1 required: lease_failed must be safely retryable and must never poison the persistent SQLite store or CrashLoop the coordinator. Make state transition + Kanban recording recoverable/idempotent with exact-run fencing; catch bounded board errors; retry lease_failed; garbage-collect only after authoritative terminal evidence; ensure one poison row cannot abort later tasks/boards. Wrap startup maintenance as safely as steady state. Add the review's end-to-end wedge/conflicting-duplicate/restart/PVC-persistence repros and fault injection for DB/Kanban errors.\n\nClose consequential handoff risks too:\n- Prevent mediator/worker RWO deadlock during drain/pressure via a self-healing placement/lifecycle design and test eviction/reschedule.\n- Make retries submit safely without updating protected/existing refs: use a unique exact-run/attempt feature ref or a broker-authorized CAS update whose policy remains human-review safe. Preserve prior work and never silently discard results.\n- Avoid repeated slow global npm installs blocking 10m Flux health; use immutable image/persistent verified tooling or decouple health with bounded readiness.\n- Rebase/reconstruct PR18 so it does NOT contain open PR14/15/16/19 as ancestors. Keep PR18's own focused diff based on current main and validate disposable synthetic merges in the documented dependency order. Force-with-lease only the existing branch after exact remote lease check.\n\nPreserve all five fixed P0/security boundaries: canonical run IDs, workspace/project/branch routing, per-ordinal crypto, ConfigMap closure, no credentials/HMAC/SA token/RBAC/SCM in model container, broker-only SCM, safe OAuth ownership, three-node schedulability, >=95% line+branch, <500 LOC. Re-run every t_dbdcd739 and t_5975c06a artifact, full quality, kustomize/kubeconform/client/server dry-runs, scheduling/drain simulation, synthetic current PR merges, diff/secret scans. Amend/push only PR18 and stop for fresh human/independent review. No merge/build publish/deploy/reconcile/live mutation/credential access.", "expected_role": "implementation", "id": "t_c31dd427", "title": "Repair PR 18 lease recovery and release isolation"} +{"board": "titan-iac", "body": "In the implementation worktree/branch, validate the completed narrow change directly rather than relying only on rendered manifests. Run all focused scheduler tests, the full relevant Hermes/Kanban regression suite, ruff/static checks, the repository-pinned Docker image build, kustomize build services/hermes, kubectl client dry-run, and git diff --check. Verify the built image contains and executes the patched scheduler behavior through the focused tests or equivalent direct image-level test path. If the tracked workflow supports authenticated internal-registry publication and publishing is necessary for PR deployability, publish only through that workflow, update the Flux manifest to the resulting immutable digest, and record it; otherwise leave a clear explicit blocker and do not invent credentials or use a public registry. Do not deploy, reconcile Flux, restart workloads, or alter live Kanban data. Return command-level pass/fail evidence, image/digest status, failures or residual risks, and any directly required corrections for the branch.", "expected_role": "implementation", "id": "t_c588a4b6", "title": "Run Hermes regression, build, and manifest validation"} +{"board": "titan-iac", "body": "Harden agent.hermes after the live Cassandra/titan-iac proof run. Work from the current private atlas/titan-iac main head and create an isolated linked worktree/branch. This is a quality-sensitive platform change. Keep it narrowly scoped to Hermes reliability; preserve unrelated work. Open a Gitea pull request for human review, but do not merge, reconcile Flux, deploy, restart the agent, or mutate live Kanban data except this task's normal lifecycle. Never expose or commit credentials.\n\nAuthoritative live evidence to encode as regressions:\n1. Three simultaneous direct CLI workers on the 4-core hermes-agent node drove load to about 45, made the hermes and oauth2-proxy containers fail probes/restart, and temporarily left the pod 8/10 Ready. Two simultaneous workers remained stable at 10/10. Set HERMES_CLI_LANE_CONCURRENCY to 2 and cap the cli-lane-runner CPU limit at 2 so UI/auth have headroom. Verify requests remain sensible.\n2. Kubernetes service-link variables contaminate worker/test environments (for example a service named hermes-claude-broker creates names that collide with tests), while the deployment already uses DNS service names. Set enableServiceLinks: false at the hermes-agent pod-spec level and add a manifest regression proving it.\n3. Scheduler correctness defect: create_task(initial_status='blocked') records a created event with status blocked but no sticky blocked event. recompute_ready() considers blocked tasks and _has_sticky_block() only recognizes blocked/unblocked events, so an explicitly blocked task with no incomplete parent auto-promotes after the dispatcher cycle. Fix this in the existing Dockerfile patching/test mechanism without weakening dependency-driven promotion or circuit-breaker semantics. Add focused RED-before/GREEN-after tests covering: explicit initial blocked stays blocked with no parents and with already-complete parents; ordinary dependency-blocked tasks promote when dependencies complete; block_task/unblock_task remains correct; circuit-breaker blocks remain sticky. Use official public behavior where possible.\n\nInspect the existing hermes-agent image patches and tests before designing the smallest robust change. Run all focused tests, the full relevant Hermes/Kanban regression suite, ruff/static checks, Docker image build using the repository's pinned Dockerfile, kustomize build services/hermes, kubectl client dry-run, and git diff --check. If the image must be published to make the PR deployable, use only the existing authenticated internal registry workflow, update the digest in Flux manifests, and report the immutable digest; do not deploy it. If publishing is not supported by the tracked workflow, leave an explicit blocker rather than inventing credentials or using a public registry.\n\nReport exact changed files, RED/GREEN evidence, tests, image/digest status, PR URL/head/base state, residual risks, and confirm the PR is open/unmerged/undeployed. Treat a merely rendered manifest as insufficient: prove the scheduler behavior directly.", "expected_role": "implementation", "id": "t_cca008de", "title": "Harden Hermes worker isolation and blocked-task semantics"} +{"board": "titan-iac", "body": "Inspect the current private atlas/titan-iac main-head manifests, pinned Dockerfile, image build/publish workflow, Flux image/digest conventions, and existing manifest tests without changing tracked files. Determine the precise hermes-agent pod-spec location for enableServiceLinks: false, the cli-lane-runner container location for CPU limit 2, and the configuration source for HERMES_CLI_LANE_CONCURRENCY=2. Identify how to add a regression that proves enableServiceLinks is false in the rendered/structured manifest and verify request settings remain appropriate. Document the supported authenticated internal-registry publication workflow, whether publishing is actually required for a deployable PR, and the exact validation commands for kustomize and kubectl client dry-run. Do not use credentials, publish images, reconcile Flux, deploy, restart workloads, or mutate live Kanban data.", "expected_role": "implementation", "id": "t_d06ecdaf", "title": "Audit Hermes workload isolation manifests and image workflow"} +{"board": "titan-iac", "body": "Perform a fresh independent read-only xhigh release/security review of the completed distributed agent.hermes worker-pool implementation produced by parent task t_af8d08d0. Read AGENTS.md and the parent result, but trust neither claimed tests nor done status. Dynamically resolve its exact pushed draft PR/head/base and review the complete diff against then-current origin/main including merged PR13 and pending PR14-PR16 compatibility. Do not edit files, mutate Kanban beyond returning result, comment, commit, push, merge, publish, deploy, reconcile, or touch live credentials.\n\nPrioritize: coordinator-only SQLite/Kanban/RWO ownership; exactly three workers on distinct eligible nodes with own RWO home/workspace; no shared mutable git; authenticated/bounded assignment, heartbeat, activity, result, retry and ack bound to board/task/run/worker ordinal/attempt/digest; replay/duplicate/stale/replacement-run/restart/partition safety; no lost accepted results; worker subscription OAuth refresh semantics without API keys; broker-only SCM and no raw Gitea token; scoped RBAC/no cluster-admin/secrets/token creation/impersonation/exec-attach-portforward/workload mutation; NetworkPolicy; SSH dedicated account compatibility; Switchyard classification/escalation; agent UI activity visibility; resource requests/anti-affinity/scheduling and RWO behavior on current nodes; rolling upgrade/rollback and coordinator/worker version skew. Run adversarial executable tests, synthetic merges with exact current PR14-PR16 heads where relevant, full focused quality gates, Kustomize/schema/client+Flux-manager server dry-runs, image/runtime static inspection, and read-only live capacity checks. Return strict SHIP or BLOCK with exact file/line and reproducible inputs. Report residual risks. Keep the review worktree pristine.", "expected_role": "review", "id": "t_dbdcd739", "title": "Independent xhigh review of distributed Hermes worker pool"} +{"board": "titan-iac", "body": "Independently review the validated branch diff against the original reliability scope and the analysis findings. Confirm scheduler tests directly prove the required blocked-task semantics; confirm manifest configuration provides two CLI lanes, a 2-CPU cli-lane-runner cap, and pod-level enableServiceLinks: false; and reject unrelated changes, credentials, deployment actions, or weakened dependency/circuit-breaker semantics. Resolve only clearly in-scope review issues, rerun affected validation if anything changes, then open a Gitea pull request from the isolated branch to the appropriate current main base for human review. Do not merge, reconcile Flux, deploy, restart the agent, or mutate live Kanban data outside normal lifecycle. Report exact changed files, RED/GREEN and full validation evidence, image and immutable digest or publication blocker, PR URL/head/base, residual risks, and explicit confirmation that the PR is open, unmerged, and undeployed.", "expected_role": "implementation", "id": "t_e3ea56cc", "title": "Review final diff and open an unmerged Gitea PR"} +{"board": "titan-iac", "body": "Perform a fresh independent read-only xhigh review of the full-handoff acceptance harness produced by parent t_6496c271. Resolve exact draft PR/head/base dynamically. Do not trust claimed coverage or go/no-go behavior. Keep worktree pristine; no edits/comments/commit/push/merge/deploy/reconcile or live mutations. Audit security and correctness of command construction/redaction before execution, environment/header/body/stdout/stderr/exception sanitization, bounded time/output/concurrency, JSON schema and deterministic overall PASS/NO_GO, mandatory NOT_RUN behavior, dual operator/in-pod evidence, revision/image freshness, partial/malformed/zero-state handling, and explicit ephemeral arming/cleanup/protected-branch rejection. Prove default mode is genuinely read-only through spies/static command inventory. Ensure it cannot read or print Vault/provider/SCM credentials and cannot falsely pass mocked/self-authored evidence. Verify every source/test file <500 LOC and >=95% per-file coverage, run mutation/adversarial tests, full relevant quality gate, Ruff/format/compile/diff/secret scans. Review dependency/merge/rollout/rollback plan against exact current PR12-PR17 and distributed-pool state. Return strict SHIP or BLOCK with executable repros/exact lines and residual risks.", "expected_role": "review", "id": "t_e7f46d44", "title": "Independent review of full-handoff acceptance harness"} +{"board": "titan-iac", "body": "Consolidate the five independent audits of the full-handoff acceptance harness (parent task t_6496c271) into one strict, decision-ready verdict.\n\nINPUTS: completed findings from (a) the security/redaction audit, (b) the read-only proof via static command inventory and spies, (c) the determinism/schema/evidence-integrity audit, (d) the executed quality gates (LOC, per-file coverage, mutation, Ruff/format/compile/diff/secret scans), and (e) the dependency/merge/rollout/rollback review. Each reports against a dynamically resolved draft PR, head SHA, and base SHA.\n\nWHAT TO DO:\n1. Confirm all five audits resolved the SAME PR number, head SHA, and base SHA. Divergence is itself a finding and the verdict must account for it.\n2. Deduplicate and reconcile findings. Where two audits disagree on the same code path, adjudicate by reading the code at the resolved head SHA yourself — do not average opinions or defer to the more confident report.\n3. Re-derive severity independently. Treat as blocking by default: any reachable mutation in default mode, any credential leak path (Vault/provider/SCM), any NOT_RUN coerced to PASS, any vacuous PASS on zero-state or partial evidence, any path where mocked or self-authored evidence passes, any file >= 500 LOC, any source file below 95% per-file coverage, any surviving mutant in verdict/redaction/read-only code, any failed lint/compile/secret scan, and any non-executable rollback. Downgrade one of these only with an explicit written justification.\n4. Verify claim coverage: every requirement in the original review scope must map to a finding or an explicit 'verified clean' statement. Anything neither verified nor refuted is an unverified gap, and unverified gaps in security or read-only enforcement force BLOCK.\n\nOUTPUT: a single verdict of exactly SHIP or BLOCK (no conditional or partial verdicts), followed by (a) blocking findings, each with exact path:line at the resolved head SHA and an executable repro command, (b) non-blocking findings, (c) residual risks that remain even if every blocker is fixed, and (d) the unverified-gap list. If BLOCK, give the minimal concrete change set that would flip it to SHIP.\n\nCONSTRAINTS: read-only throughout — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations; worktree stays pristine. Do not accept any upstream audit's conclusion at face value where you can cheaply verify it against the code yourself.", "expected_role": "review", "id": "t_fa77b785", "title": "Synthesize independent review into a strict SHIP or BLOCK verdict"} diff --git a/testing/tests/data/hermes_t_dbdcd739_context.txt b/testing/tests/data/hermes_t_dbdcd739_context.txt new file mode 100644 index 00000000..f3eee4ed --- /dev/null +++ b/testing/tests/data/hermes_t_dbdcd739_context.txt @@ -0,0 +1,73 @@ +# Kanban task t_dbdcd739: Independent xhigh review of distributed Hermes worker pool + +Assignee: cli-claude-xhigh +Status: blocked +Workspace: worktree @ /opt/data/workspace/projects/titan-iac/.worktrees/t_dbdcd739 +Max runtime: 21600s +Terminal timeout: 21570s +Branch: review/hermes-distributed-worker-pool + +## Body +Perform a fresh independent read-only xhigh release/security review of the completed distributed agent.hermes worker-pool implementation produced by parent task t_af8d08d0. Read AGENTS.md and the parent result, but trust neither claimed tests nor done status. Dynamically resolve its exact pushed draft PR/head/base and review the complete diff against then-current origin/main including merged PR13 and pending PR14-PR16 compatibility. Do not edit files, mutate Kanban beyond returning result, comment, commit, push, merge, publish, deploy, reconcile, or touch live credentials. + +Prioritize: coordinator-only SQLite/Kanban/RWO ownership; exactly three workers on distinct eligible nodes with own RWO home/workspace; no shared mutable git; authenticated/bounded assignment, heartbeat, activity, result, retry and ack bound to board/task/run/worker ordinal/attempt/digest; replay/duplicate/stale/replacement-run/restart/partition safety; no lost accepted results; worker subscription OAuth refresh semantics without API keys; broker-only SCM and no raw Gitea token; scoped RBAC/no cluster-admin/secrets/token creation/impersonation/exec-attach-portforward/workload mutation; NetworkPolicy; SSH dedicated account compatibility; Switchyard classification/escalation; agent UI activity visibility; resource requests/anti-affinity/scheduling and RWO behavior on current nodes; rolling upgrade/rollback and coordinator/worker version skew. Run adversarial executable tests, synthetic merges with exact current PR14-PR16 heads where relevant, full focused quality gates, Kustomize/schema/client+Flux-manager server dry-runs, image/runtime static inspection, and read-only live capacity checks. Return strict SHIP or BLOCK with exact file/line and reproducible inputs. Report residual risks. Keep the review worktree pristine. + +## Prior attempts on this task +### Attempt 1 — blocked (cli-claude-xhigh, 2026-08-17 10:02, 4h ago) +Manual shepherd stop: review completed BLOCK three times with durable evidence, but goal judge incorrectly demanded implementation repairs and entered a costly continuation loop. Evidence preserved under /opt/data/workspace/evidence/t_dbdcd739; repair belongs to a separate implementation task. + +## Parent task results +_Handoffs from upstream tasks, captured when each parent completed (see age below). These are point-in-time snapshots, not live state — if a result drives your current work and it's not recent, re-verify against the source before acting on it as current._ +### t_af8d08d0 (completed 4h ago) +Implemented and pushed the safe three-node agent.hermes execution pool. PR #18 is verified open and draft at the exact pushed SHA, with one coordinator, three ordinal-isolated RWO workers, authenticated/idempotent run fencing, subscription OAuth handling, narrow SCM/RBAC/network boundaries, visible activity, retention controls, and adversarial coverage. +_metadata_: `{"artifacts": ["/opt/data/workspace/projects/titan-iac/.worktrees/t_af8d08d0/build/junit-hermes-pool-final.xml"], "blockers": [], "changed_files": ["clusters/atlas/flux-system/applications/hermes/kustomization.yaml", "services/hermes/execution-coordinator-patch.yaml", "services/hermes/execution-worker-networkpolicy.yaml", "services/hermes/execution-worker-rbac.yaml", "services/hermes/execution-worker-statefulset.yaml", "services/hermes/kustomization.yaml", "services/hermes/scripts/execution_pool_askpass.sh", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/stage_runtime_access.py", "services/hermes/service.yaml", "services/vault/scripts/vault_k8s_auth_configure.sh", "testing/tests/test_hermes_auto_router.py", "testing/tests/test_hermes_execution_pool.py", "testing/tests/test_hermes_execution_pool_assignment.py", "testing/tests/test_hermes_runtime_access.py"], "classifier": "switchyard-classifier", "claude_session_id": null, "codex_thread_id": "01a00eeb-0327-7962-995e-1e1f235983ed", "effort": "high", "executor": "direct-cli-lane", "findings": ["The repository-wide local quality profile retains an unrelated baseline hygiene failure for four pre-existing test files over 500 lines. Docs, Ruff, unit, and coverage portions pass; all new production files are below 500 lines and the primary adversarial test file is exactly 500 lines.", "No deployment, Flux reconciliation, image publication, workload mutation, or live three-node scheduling proof was performed, as required."], "goal_judge_reason": "The worker report provides concrete evidence that all explicit requirements have been met. The implementation includes the creation of a coordinator and three ordinal-isolated RWO workers, authenticated/idempotent run fencing, subscription OAuth handling, narrow SCM/RBAC/network boundaries, visible activity, retention controls, and adversarial coverage. All tests are green, including unit tests, strict quality checks, production file scans, Kustomize renders, Flux dry-runs, and synthetic merges with open draft PRs. The report also confirms that no deployment or live three-node scheduling proof was performed as required.", "goal_mode": true, "goal_turn": 1, "model": "gpt-5.6-sol", "provider": "codex", "state_file": "/opt/data/cli-lanes/titan-iac/t_af8d08d0.json", "tests_run": ["Configured unit suite: 397 passed", "Final focused pool/lane/runtime/router suite: 150 passed", "Strict Ruff, compileall, sh -n, ShellCheck, and git diff --check: passed", "Production changed-file detect-secrets scan: 0 findings", "Hermes and Vault Kustomize renders: passed", "Hermes and Vault Flux build dry-runs: passed", "Hermes and Vault kubectl client and server-side dry-runs: passed", "PR #14 updated synthetic merge: conflict-free, rendered, 61 tests passed", "PR #15 updated synthetic merge: conflict-free, rendered, 62 tests passed", "PR #16 updated synthetic merge: conflict-free, rendered, 62 tests passed", "PR #17 synthetic merge: conflict-free, rendered, 62 tests passed", "Remote branch and PR head SHA verification: exact match"]}` + +## Recent work by @cli-claude-xhigh +- t_5975c06a — Independent re-review of repaired PR 18 distributed pool (2026-08-17 14:19, 11m ago): Independent read-only xhigh re-review of draft PR #18 (feature/hermes-distributed-worker-pool) at exact head 689bcb6e48414854447988ba174143a307d308b9 against current remote main d8f2d818b9a552ea6c2d7f +- t_5ae177fc — Independent re-review of repaired PR 19 acceptance harness (2026-08-17 13:52, 38m ago): Independent read-only xhigh re-review of draft PR #19 at exact head c808baff40a12a90f43c2e5e39b94d139f88ea29 against current remote main d8f2d818b9a552ea6c2d7fe86554be829bd5ffff (merge-base 30259b52, +- t_261e208f — Independent review of cli-lane process reaping (2026-08-17 11:34, 2h ago): Independent read-only release review of atlas/titan-iac PR #20 (feature/hermes-cli-process-reaping) at head a242dcc786576ae1a18000cb4941836ff610dff2 vs base main d8f2d818b9a552ea6c2d7fe86554be829bd5ff +- t_6496c271 — Build fail-closed Hermes full-handoff acceptance harness (2026-08-17 10:17, 4h ago): Built a fail-closed, read-only-by-default Hermes full-handoff acceptance harness (71 checks, 15 new production modules under scripts/ops/, all <500 LOC, 100% per-file line coverage), a 12-module test +- t_20f2fd16 — Repair Hermes PR 13 producer-drift regression coverage (2026-08-16 22:46, 15h ago): Amended PR #13 with commit 750dfa24 (pushed to wt/t_cca008de), fixing the single P1 review finding from t_06771c95 and applying the operator recovery addendum. + +## Comment thread +comment from worker `cli-lane-runner` at 2026-08-17 10:02, 4h ago: +CLI route: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh' + +comment from worker `cli-lane-runner` at 2026-08-17 10:28, 4h ago: +Goal completion rejected; continuing turn 2/8: local goal judge requested continuation: The review identified several critical defects that need to be addressed. Specifically, there are five P0 defects which include issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective. + +comment from worker `cli-lane-runner` at 2026-08-17 10:28, 4h ago: +Goal route 2/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh' + +comment from worker `codex-shepherd` at 2026-08-17 10:29, 4h ago: +Reviewer semantics correction: this is a read-only review. A completed review with a BLOCK verdict and concrete findings fully satisfies the task; do not implement fixes in the review worktree. Return status=completed, put release blockers in findings, blockers=[] unless an obstacle prevents reviewing. The local goal judge must not demand that a reviewer repair the reviewed PR. + +comment from worker `cli-lane-runner` at 2026-08-17 10:36, 3h ago: +Goal completion rejected; continuing turn 3/8: local goal judge requested continuation: The report identifies several critical defects that need to be addressed, including issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective. + +comment from worker `cli-lane-runner` at 2026-08-17 10:36, 3h ago: +Goal route 3/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh' + +comment from worker `cli-lane-runner` at 2026-08-17 10:40, 3h ago: +Goal completion rejected; continuing turn 4/8: local goal judge requested continuation: The report identifies several critical defects that need to be addressed, including issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective. + +comment from worker `cli-lane-runner` at 2026-08-17 10:40, 3h ago: +Goal route 4/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh' + +comment from worker `cli-lane-runner` at 2026-08-17 10:42, 3h ago: +Provider fallback: claude -> codex; Jetson reclassified the retry boundary. + +comment from worker `cli-lane-runner` at 2026-08-17 10:43, 3h ago: +Goal completion rejected; continuing turn 5/8: The live repository instructions override the generic connector path. I found no repo-local AGENTS.md; /opt/data/workspace/AGENTS.md is authoritative for this worktree. + +comment from worker `cli-lane-runner` at 2026-08-17 10:43, 3h ago: +Goal route 5/8: codex/gpt-5.6-sol at xhigh; classifier=switchyard-manual-health-guard; Switchyard selected excluded claude; preserved its xhigh effort on healthy-provider route. random routing selected target 'worker/codex/sol/xhigh' + +comment from worker `default` at 2026-08-17 10:44, 3h ago: +BLOCKED: Manual shepherd stop: review completed BLOCK three times with durable evidence, but goal judge incorrectly demanded implementation repairs and entered a costly continuation loop. Evidence preserved under /opt/data/workspace/evidence/t_dbdcd739; repair belongs to a separate implementation task. + +comment from worker `cli-lane-runner` at 2026-08-17 10:44, 3h ago: +Goal completion rejected; continuing turn 6/8: I’m continuing the review; the last metadata query was interrupted before returning data. I’ll now resolve the Gitea PR state with the mandated helper and independently rerun the inherited blockers and release gates. + +comment from worker `cli-lane-runner` at 2026-08-17 10:44, 3h ago: +Goal route 6/8: codex/gpt-5.6-sol at xhigh; classifier=switchyard-manual-health-guard; Switchyard selected excluded claude; preserved its xhigh effort on healthy-provider route. random routing selected target 'worker/codex/sol/xhigh' diff --git a/testing/tests/data/hermes_t_dbdcd739_result.json b/testing/tests/data/hermes_t_dbdcd739_result.json new file mode 100644 index 00000000..a5e2404c --- /dev/null +++ b/testing/tests/data/hermes_t_dbdcd739_result.json @@ -0,0 +1,47 @@ +{ + "status": "completed", + "summary": "Independent read-only xhigh release/security review of PR #18 (atlas/titan-iac, head 20002527512235b29054f3636b02418168373009, base origin/main d8f2d818b9a552ea6c2d7fe86554be829bd5ffff which contains merged PR #13 ab346f55). Verdict: BLOCK. Five P0 defects are executably reproducible, four of which fire on the coordinator's first tick: (1) Coordinator.finalize() compares an int Kanban current_run_id to a str run_id so every accepted result is discarded while the worker is told accepted=True, and the resulting PK collision makes reconcile() throw every cycle; (2) the workspace_path gate force-blocks every live Kanban task including the two currently running ones; (3) the branch regex rejects the established wt/ convention; (4) Task has no repo_url so every board's tasks are pushed to titan-iac; (5) the pool ConfigMap does not ship the 14 modules PR #15 splits cli_lane_runner into, so coordinator and all three workers fail at import after a clean textual merge. P1 findings cover control-plane node placement, capacity/affinity steering onto the routing model and Flux controllers, model-container access to its own signing/SCM boundaries, worker poisoning of the shared Vault credential store, and bypass of PR #14's SCM authority boundary. Architecture that is sound (coordinator-only Kanban/SQLite, per-ordinal RWO PVCs, no shared mutable git, zero worker RBAC, worker-to-worker isolation, HMAC envelope binding) is called out explicitly. Two observed failures were confirmed pre-existing and unrelated to the PR. The review worktree was restored to review/hermes-distributed-worker-pool @ 0dd6ea0f and is clean including ignored paths.", + "changed_files": [], + "tests_run": [ + "Focused pool suite at PR head: pytest testing/tests/test_hermes_execution_pool.py test_hermes_execution_pool_assignment.py test_hermes_runtime_access.py test_hermes_auto_router.py -> 82 passed", + "Full suite at PR head (PYTHONPATH=.): pytest testing/tests -> 1 failed, 332 passed (failure pre-existing and environment-sensitive)", + "Repo quality gate: python -m testing.quality_gate --profile local -> 1 failed, 396 passed (same pre-existing failure); ruff check on all changed .py -> All checks passed", + "compileall on all changed .py -> OK; sh -n services/hermes/scripts/execution_pool_askpass.sh -> OK; git diff --check ab346f55..20002527 -> OK", + "Adversarial repro repro_finalize_stale.py -> worker ack accepted=True, complete_task=[] block_task=[], store state=stale had_result=True, all 3 ordinals freed, reconcile() raises ProtocolError: conflicting duplicate assignment", + "Adversarial repro repro_workspace_block.py against live titan-iac board row (mode=ro&immutable=1) -> dispatch() and reconcile() both emit kanban block_task; control case with workspace_path cleared still blocks on branch policy", + "Adversarial repro repro_configmap_import.py over synthetic merges -> merge-main/pr14/pr16 import rc=0; merge-pr15 rc=1 ModuleNotFoundError: No module named 'cli_lane_board' for both execution_pool_coordinator and execution_pool_worker", + "Synthetic merges via git merge-tree --write-tree of PR18 head against origin/main d8f2d818 and PR14 3cd8a17e772978fb554d88dd1c59777786be3991, PR15 7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e, PR16 5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a, PR17 48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c, PR19 8f005458282269ba5c07941814e4237f5d4cf3ac -> all CLEAN; each materialized tree ran the focused pool suite (81-82 passed) and kustomize build services/hermes (OK)", + "kustomize build services/hermes and services/vault -> OK; flux build kustomization hermes --dry-run -> exit 0", + "kubectl apply --dry-run=client -> only the pre-existing >256KiB hermes-coordinator annotation error; kubectl apply --dry-run=server --server-side --field-manager=kustomize-controller -> clean (Flux path)", + "Read-only live cluster checks: k3s v1.33.3, node labels/taints, apps.kubernetes.io/pod-index availability, astreae StorageClass (Longhorn, Immediate, diskSelector=astreae, 24.6TiB free vs 270GiB needed), per-node free CPU/memory feasibility for the 1150m/2304Mi worker pod, vault pod labels, existing hermes NetworkPolicies, RBAC bindings in the render, image digest drift" + ], + "artifacts": [ + "/opt/data/workspace/evidence/t_dbdcd739/RUN.md", + "/opt/data/workspace/evidence/t_dbdcd739/repro_finalize_stale.py", + "/opt/data/workspace/evidence/t_dbdcd739/repro_workspace_block.py", + "/opt/data/workspace/evidence/t_dbdcd739/repro_configmap_import.py" + ], + "findings": [ + "P0-1 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:248: finalize() evaluates `_task_value(task, \"current_run_id\") != binding[\"run_id\"]` without coercion. hermes_cli/kanban_db.py:853 and the schema at :1108 declare current_run_id as INTEGER, and live rows confirm typeof=integer (titan-iac t_dbdcd739 -> 23), while binding['run_id'] is a str produced by dispatch() at :386 and stored in a TEXT column. int 23 != str '23' is always True, so every result is marked stale: complete_task/block_task are never called, the Kanban task stays 'running' forever, and the worker receives accepted=True and writes terminal_at. Repro repro_finalize_stale.py output: ack {'accepted': True, 'duplicate': False}; complete_task []; block_task []; store state=stale had_result=True; free ordinals [0,1,2]. Follow-on: the row survives under PK (board,task_id,run_id) with a different payload digest, so reconcile() at :358 raises ProtocolError 'conflicting duplicate assignment' (execution_pool_protocol.py:307) on every 5s cycle, aborting the entire reconcile pass for all boards. Zero test coverage: the only tests touching Coordinator.finalize stub it (testing/tests/test_hermes_execution_pool_assignment.py:71 `pool.finalize = called.append`), and grep for current_run_id in both new test files returns nothing.", + "P0-2 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:65-69: resolve_scm raises RuntimeError whenever task.workspace_path is non-empty; dispatch() (:396-406) and reconcile() (:347-356) convert that into kanban_db.block_task(kind='capability'). A read-only census of /opt/data/kanban/boards/*/kanban.db shows every task on titan-iac, cassandra and soteria carries workspace_path. On first coordinator tick this force-blocks the two currently running titan-iac tasks (t_6496c271 and t_dbdcd739, this review) and the two ready cli tasks (t_83252d81, t_87cd9076). Repro repro_workspace_block.py cases [1] and [2].", + "P0-3 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:40,77: BRANCH = ^(?:feature|fix|chore|docs|test|refactor)/... rejects the branch names Hermes itself writes. hermes_cli/kanban_db.py:5447, :7275 and :7367 default worktree tasks to f'wt/{task.id}', and live rows carry wt/t_1e95ea6d as well as review/... names (t_dbdcd739 -> review/hermes-distributed-worker-pool). Repro repro_workspace_block.py case [3]: with workspace_path cleared the same task still blocks with 'task feature branch is outside the SCM branch policy'.", + "P0-4 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:70,79-81: remote and base_branch come from _task_value(task, 'repo_url') / 'base_branch', but the hermes_cli.kanban_db.Task dataclass (kanban_db.py:838-916) defines neither field, so getattr always returns the DEFAULT_REPO https://scm.bstein.dev/atlas/titan-iac.git. Every task on every board is therefore cloned from and pushed to titan-iac. The metis scratch task t_8569b7b9 - the one live task that survives P0-2 - would run against a titan-iac checkout and push to feature/hermes-t_8569b7b9 there.", + "P0-5 BLOCKER - services/hermes/kustomization.yaml:64-78 is runtime-incompatible with open PR #14... correction, PR #15 fix/hermes-result-decomposition-reliability head 7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e, despite a clean textual merge. PR15 decomposes cli_lane_runner.py into 14 sibling modules (cli_lane_board, cli_lane_config, cli_lane_dispatch, cli_lane_evidence, cli_lane_execution, cli_lane_files, cli_lane_finalization, cli_lane_prompt, cli_lane_provider, cli_lane_quarantine, cli_lane_records, cli_lane_recovery, cli_lane_retention, cli_lane_routing), while the hermes-execution-pool ConfigMap ships only cli_lane_goal.py and cli_lane_runner.py. Proof repro_configmap_import.py: merge-main/pr14/pr16 import rc=0, merge-pr15 rc=1 ModuleNotFoundError: No module named 'cli_lane_board' for both execution_pool_coordinator and execution_pool_worker -> coordinator plus all three workers CrashLoopBackOff.", + "P1 - services/hermes/execution-worker-statefulset.yaml:87-90: the second nodeSelectorTerm requires only arch=arm64, hardware=rpi5 and a hostname NotIn list, dropping the node-role.kubernetes.io/worker In ['true'] requirement that agent-deployment.yaml:109-111 uses. Measured against live capacity: 8 nodes satisfy the affinity, only 4 can fit the 1150m CPU / 2304Mi pod (titan-20 1270m free, titan-21 1520m, titan-0a 3730m, titan-0c 3630m), and titan-0a/titan-0c are k3s control-plane/master. podAntiAffinity forces 3 distinct hosts, so one worker necessarily lands on a control-plane node while running codex --dangerously-bypass-approvals-and-sandbox / claude --dangerously-skip-permissions (cli_lane_runner.py:594, :631).", + "P1 - services/hermes/execution-worker-statefulset.yaml:91-99: preferredDuringScheduling weight 100 on node-role.kubernetes.io/accelerator steers two of three workers onto titan-20 (which runs ai/ollama, the Jetson routing model behind Switchyard AUTO) and titan-21 (which runs flux-system source-controller, image-automation-controller, image-reflector-controller and notification-controller). titan-20 retains only ~120m CPU slack after placement while the worker container's limits are 5 CPU / 6Gi, so the pool contends directly with the routing model and the GitOps control plane.", + "P1 - services/hermes/scripts/execution_pool_client.py:142 and execution_pool_scm.py:442 bind 0.0.0.0 (required for kubelet httpGet probes). The model container holds no key but shares the pod network namespace, so it can POST 127.0.0.1:9009 {'operation':'poll'} to obtain a freshly coordinator-signed assignment envelope and replay it to 127.0.0.1:9008 /v1/scm {'operation':'submit'} and to 9009 {'operation':'result'}. Authority stays bounded to its own board/task/run/ordinal by ClientBoundary.forward (client.py:84) and _binding (scm.py:123), and worker-to-worker reach is blocked by ingress: [], but it bypasses the worker's own gates: the uncommitted/untracked check (execution_pool_worker.py:332-337) and the commits-ahead check (:342).", + "P1 - The model container has read-write access to /runtime-access/claude/.credentials.json and /runtime-access/codex/auth.json (execution-worker-statefulset.yaml:206), and the credential-sync sidecar CAS-writes whatever it finds there into kv/data/atlas/hermes/agent-tokens (sync_runtime_credentials.py:105-116) using role hermes-execution-credential-sync, which services/vault/scripts/vault_k8s_auth_configure.sh:228-229 grants create/update/read on that path. That secret backs the coordinator's own credentials and is the derivation input for the pool HMAC key (printf \"hermes-execution-pool-v1:%s\" .Data.data.agent_api_key | sha256sum). Separately, all three workers plus the agent now refresh the same subscription OAuth documents independently; provider-side refresh-token rotation makes this a lost-update/lockout race that CAS cannot resolve.", + "P1 - Open PR #14 feature/hermes-safe-gitea-pr-client (head 3cd8a17e772978fb554d88dd1c59777786be3991) deletes services/hermes/scripts/gitea_api.py and gitea_askpass.sh and introduces a dedicated hermes-scm-broker (own namespace, NetworkPolicy, gitea_api_policy.py, DLP tests) to bound Atlas PR authority. PR18 concurrently stages the raw kv/data/atlas/hermes/developer-gitea token into three new pods (stage_runtime_access.py:192), authenticates git with it (execution_pool_askpass.sh), calls the Gitea REST API directly (execution_pool_scm.py:355-373) and opens the egress path (execution-worker-networkpolicy.yaml:37-42). Verified on the merged tree: merge-pr14 still stages the raw token, so the broker is bypassed for the highest-volume automated PR path and the token blast radius widens by three pods on three more nodes.", + "P2 - services/hermes/scripts/execution_pool_protocol.py:227-229,337,385: lease_seconds, lease_until and last_heartbeat are written and never read anywhere in the pool. There is no lease-expiry reaper, so a worker that dies mid-assignment holds its ordinal until Kanban's own DEFAULT_CLAIM_TTL (7 days, cli_lane_runner.py:40) causes a divergence that reconcile() can observe.", + "P2 - services/hermes/scripts/execution_pool_worker.py:407: main() swallows OSError/RuntimeError/ValueError, sleeps 10s and re-polls the identical assignment; attempt is never incremented and Kanban is never informed. Reachable today: cli_lane_runner.parse_assignee (:152) raises ValueError for cli-codex-sol-xhigh, a live assignee on two titan-iac tasks, so such an assignment spins forever holding its ordinal. The legacy path blocked the task instead (cli_lane_runner.py:832-848).", + "P2 - services/hermes/kustomization.yaml:77-78 sets disableNameSuffixHash: true on hermes-execution-pool, and neither execution-worker-statefulset.yaml nor execution-coordinator-patch.yaml carries a config-revision annotation (agent-deployment.yaml:28 uses ai.bstein.dev/config-rev for exactly this). Pool code changes therefore never trigger a rollout; coordinator and workers pick up new code only on unrelated restarts, producing mixed-version windows against a strict contract that has no negotiation (execution_pool_protocol.py:177 rejects any envelope whose field set differs from version 1).", + "P2 - services/hermes/execution-worker-networkpolicy.yaml:33-36 allows worker egress to hermes-model-gate:8080, but services/hermes/networkpolicy.yaml:26-41 (hermes-model-gate-ingress) does not list hermes-execution-worker and nothing in the worker path calls model-gate (select_route targets Switchyard only). The rule is dead and asymmetric; remove it or complete the ingress side.", + "P2 - services/hermes/execution-worker-statefulset.yaml:303 mounts the whole runtime-access volume read-write into credential-sync, whereas the agent mounts only the claude and codex subPaths read-only (agent-deployment.yaml:951-952). Minor hardening regression relative to the established pattern.", + "PRE-EXISTING, not caused by PR18 - testing/tests/test_hermes_coordinator.py::test_cassandra_sync_repairs_origin_without_token fails wherever /runtime-access/gitea-token exists, because the test never patches hermes_coordinator._gitea_token_available (hermes_coordinator.py:148-155). Neither the module nor the test is in the PR diff and it fails standalone. This accounts for the 1 failure in both the full suite (332 passed) and the quality gate (396 passed), versus the parent task's claim of 397 passed in an environment without that file.", + "PRE-EXISTING, not caused by PR18 - kubectl apply --dry-run=client rejects ConfigMap hermes-coordinator with 'metadata.annotations: Too long: may not be more than 262144 bytes'. Measured generator input is 341438 bytes at base ab346f55 and 343841 at head, so the threshold was already exceeded before this PR. Flux uses server-side apply and the SSA dry-run with field-manager kustomize-controller is clean.", + "NOTE - ruff format --check would reformat 10 of the changed files, but testing/quality_gate.py:228 runs only ruff check, so this is unenforced style drift rather than a gate failure.", + "NOTE - git/cluster image drift: both origin/main and PR18 pin hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107, while the live hermes-agent pod runs sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7. Pre-existing drift, but it means the new workers and the currently running coordinator would execute different image builds until Flux recreates the agent. The pinned digest is cached only on titan-11/15/19, none of the four candidate worker nodes, so first scheduling incurs a cold pull.", + "VERIFIED SOUND (no defect) - coordinator-only Kanban/SQLite ownership (workers mount no /opt/data; hermes-agent is replicas:1 with strategy Recreate); per-ordinal RWO 30Gi Longhorn PVCs via volumeClaimTemplates with ample astreae capacity (24624Gi schedulable free vs 270Gi needed across 87 existing volumes); no shared mutable git (per-run clone under /workspace/runs///); zero Kubernetes RBAC for the hermes-execution-worker service account (no RoleBinding or ClusterRoleBinding in the render, automountServiceAccountToken: false, Vault-audience token projected only into credential-sync); worker-to-worker isolation via ingress: [] with kubelet probes proven working under the identical pattern on hermes-chat-sandbox; HMAC envelope binding to board/task/run/worker ordinal/attempt/payload digest with expiry, clock-skew bounds, delivery-id replay guard and a partial unique index enforcing one live assignment per ordinal; result idempotency by digest; heartbeat lease loss terminates the provider process (cli_lane_runner.py:46-50); no node-SSH material staged to workers (stage_runtime_access.py:179-204); Switchyard is the only routing path; heartbeat activity plus a deduplicated terminal batch reach the Kanban worker log for agent UI visibility." + ], + "blockers": [] +} \ No newline at end of file diff --git a/testing/tests/test_hermes_cli_lane_goal.py b/testing/tests/test_hermes_cli_lane_goal.py index 24bea93d..9e1251fa 100644 --- a/testing/tests/test_hermes_cli_lane_goal.py +++ b/testing/tests/test_hermes_cli_lane_goal.py @@ -44,13 +44,13 @@ def _result(**overrides): ], ) def test_unfinished_completion_evidence_is_rejected(result): - assert goal.unfinished_result_reason(result) + assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) def test_completed_pending_state_test_name_is_not_a_false_positive(): result = _result(tests_run=["pytest tests/test_pending_build_state.py: 8 passed"]) - assert goal.unfinished_result_reason(result) is None + assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None def test_completed_review_findings_are_not_task_blockers(): @@ -59,7 +59,7 @@ def test_completed_review_findings_are_not_task_blockers(): findings=["A casting-table variant fails open."], ) - assert goal.unfinished_result_reason(result) is None + assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None class JudgeResponse: diff --git a/testing/tests/test_hermes_cli_review_contract.py b/testing/tests/test_hermes_cli_review_contract.py new file mode 100644 index 00000000..ab6237de --- /dev/null +++ b/testing/tests/test_hermes_cli_review_contract.py @@ -0,0 +1,313 @@ +"""Upgrade, single-shot and redaction contracts for the Hermes goal gate.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SPEC = importlib.util.spec_from_file_location( + "cli_lane_review_contract_test", + ROOT / "services/hermes/scripts/cli_lane_goal.py", +) +assert SPEC and SPEC.loader +goal = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = goal +SPEC.loader.exec_module(goal) + +# A synthetic 40-character lowercase hex string. It has the shape of a Gitea +# personal access token *and* of a git SHA-1; no real credential is read here. +SYNTHETIC_HEX = "0123456789abcdef" * 2 + "01234567" +REAL_SHA = "b080b5f622ae0998213f3287762aea30dc931a73" + +REPORTS = { + "block_review": { + "status": "completed", + "summary": ( + "Independent read-only review of PR #18. Verdict: BLOCK. Deployment " + "verification is pending on two of three nodes, which is exactly why " + "the reviewed change is unfit to ship." + ), + "changed_files": [], + "tests_run": ["pytest testing/tests: 337 passed"], + "artifacts": [], + "findings": ["P0 - coordinator.py:248 compares int to str run ids."], + "blockers": [], + }, + "bare_token_review": { + "status": "completed", + "summary": ( + "Independent read-only review of PR #18 concludes BLOCK because the " + "coordinator drops a lease on its first tick and deployment " + "verification is pending on two of three nodes." + ), + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": ["P0 - the lease is dropped before the first heartbeat."], + "blockers": [], + }, + "terse_ship_audit": { + "status": "completed", + "summary": "Decision: ship. Audit done.", + "changed_files": [], + "tests_run": ["ruff check: clean"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + "block_audit_without_findings": { + "status": "completed", + "summary": ( + "The generated-strategy audit harness was inspected end to end and " + "the verdict is BLOCK on the release train for now." + ), + "changed_files": [], + "tests_run": ["pytest -q: 61 passed"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + "finished_implementation": { + "status": "completed", + "summary": "Every acceptance criterion passed and the branch was pushed.", + "changed_files": ["services/hermes/scripts/cli_lane_goal.py"], + "tests_run": ["pytest -q: 12 passed"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + "unfinished_implementation": { + "status": "completed", + "summary": "The broad rerun remains active before the final push.", + "changed_files": ["services/hermes/scripts/cli_lane_goal.py"], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + "incomplete_turn": { + "status": "incomplete", + "summary": "Only two of six boundaries were read.", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [], + }, + "completed_with_blockers": { + "status": "completed", + "summary": "The review finished but the head could not be resolved.", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": ["the Gitea API was unreachable"], + }, +} + + +def _base_unfinished_result_reason(result): + """The completion gate exactly as main d8f2d818 shipped it. + + Kept as a local oracle so the upgrade properties below are checked against + the previous semantics rather than against the implementation under test. + """ + status = str(result.get("status") or "") + summary = str(result.get("summary") or "").strip() + blockers = result.get("blockers") + if status == "incomplete": + return summary or "worker explicitly reported incomplete work" + if status != "completed": + return None + if isinstance(blockers, list) and any(str(item).strip() for item in blockers): + return "worker reported blockers while claiming completion" + tests = result.get("tests_run") + evidence = [summary] + if isinstance(tests, list): + evidence.extend(str(item) for item in tests) + match = goal.UNFINISHED_EVIDENCE.search("\n".join(evidence)) + if match: + return f"completion evidence says work is unfinished: {match.group(0).strip()}" + return None + + +def _accepted(result, *, role): + return goal.unfinished_result_reason(result, role=role) is None + + +@pytest.mark.parametrize("name", sorted(REPORTS)) +def test_a_role_blind_replay_accepts_whatever_the_lane_accepted(name): + """PR15 re-runs this gate on journalled terminal records without a card. + + A record accepted by any version of the lane must stay valid, or an upgrade + quarantines it and re-dispatches an already-accepted task. + """ + result = REPORTS[name] + accepted_somewhere = _base_unfinished_result_reason(result) is None or any( + _accepted(result, role=role) + for role in (goal.REVIEW_ROLE, goal.IMPLEMENTATION_ROLE) + ) + + if accepted_somewhere: + assert goal.unfinished_result_reason(result) is None + + +def test_role_blind_replay_keeps_the_unfinished_work_integrity_check(): + """PR15's ``_terminal_record_valid`` relies on this without a card.""" + assert goal.unfinished_result_reason(REPORTS["completed_with_blockers"]) + assert goal.unfinished_result_reason(REPORTS["incomplete_turn"]) + assert goal.unfinished_result_reason(REPORTS["unfinished_implementation"]) + assert goal.unfinished_result_reason( + {**REPORTS["terse_ship_audit"], "summary": "tests are still running"} + ) + + +def test_role_blind_replay_exempts_only_a_complete_review_deliverable(): + """The exemption is exactly the shape the role-aware lane calls a review.""" + complete = REPORTS["bare_token_review"] + mutated = {**complete, "changed_files": ["src/a.py"]} + verdictless = {**complete, "summary": complete["summary"].replace("BLOCK", "no")} + + assert _base_unfinished_result_reason(complete) + assert _accepted(complete, role=goal.REVIEW_ROLE) + assert goal.unfinished_result_reason(complete) is None + assert goal.unfinished_result_reason(mutated) + assert goal.unfinished_result_reason(verdictless) + + +@pytest.mark.parametrize("name", sorted(REPORTS)) +def test_implementation_completion_is_byte_for_byte_the_previous_gate(name): + result = REPORTS[name] + + assert goal.unfinished_result_reason( + result, role=goal.IMPLEMENTATION_ROLE + ) == ( + goal.sanitize_reason(_base_unfinished_result_reason(result)) + if _base_unfinished_result_reason(result) + else None + ) + + +def test_the_two_single_shot_outcomes_that_change_are_the_documented_ones(): + """Single-shot runs this gate as its only completion check. + + The verdict contract binds there too, so a review's own deliverable decides + the outcome rather than prose about the artifact it reviewed. + """ + now_completes = REPORTS["bare_token_review"] + now_blocks = (REPORTS["terse_ship_audit"], REPORTS["block_audit_without_findings"]) + + assert _base_unfinished_result_reason(now_completes) + assert _accepted(now_completes, role=goal.REVIEW_ROLE) + for result in now_blocks: + assert _base_unfinished_result_reason(result) is None + problem = goal.unfinished_result_reason(result, role=goal.REVIEW_ROLE) + assert problem and problem.startswith(goal.READ_ONLY_GUARD) + + +@pytest.mark.parametrize( + ("text", "secret"), + [ + ( + f"cloned https://hermes:{SYNTHETIC_HEX}@scm.bstein.dev/atlas/x.git", + SYNTHETIC_HEX, + ), + (f"exported GITEA_TOKEN={SYNTHETIC_HEX} into the lane", SYNTHETIC_HEX), + (f"the pat {SYNTHETIC_HEX} was reused by the askpass helper", SYNTHETIC_HEX), + (f"Authorization: token {SYNTHETIC_HEX}", SYNTHETIC_HEX), + ("token ghp_" + "A" * 24, "ghp_" + "A" * 24), + ("the secret is xoxb-1234567890-abcdefghij", "xoxb-1234567890-abcdefghij"), + (f"reviewed exact head {REAL_SHA} against base main", None), + ], +) +def test_judge_reasons_redact_credentials_but_keep_commit_evidence(text, secret): + """A 40-hex PAT and a 40-hex commit SHA are the same string in isolation. + + Redaction is anchored on the surrounding syntax so credentials disappear + and the exact head a review pins its findings to survives as evidence. + """ + reason = goal.sanitize_reason(text) + + if secret is None: + assert reason == text + else: + assert secret not in reason + assert "[redacted]" in reason or "***" in reason + + +def test_redaction_falls_back_when_the_agent_runtime_is_absent(monkeypatch): + monkeypatch.setattr(goal, "_canonical_redact", None) + + reason = goal.sanitize_reason( + f"token ghp_{'A' * 24} and https://ci:{SYNTHETIC_HEX}@scm.bstein.dev/x.git" + ) + + assert "ghp_" not in reason + assert SYNTHETIC_HEX not in reason + assert reason.count("[redacted]") == 2 + + +def test_redaction_delegates_to_the_canonical_helper_when_it_is_installed(monkeypatch): + seen = [] + + def spy(text, **kwargs): + seen.append(kwargs) + return text.replace("OPAQUE", "***") + + monkeypatch.setattr(goal, "_canonical_redact", spy) + + assert goal.sanitize_reason("value OPAQUE here") == "value *** here" + assert seen == [{"force": True}] + + +def test_the_canonical_redactor_is_the_one_the_lane_runtime_ships(): + canonical = pytest.importorskip("agent.redact") + + assert goal._canonical_redact is canonical.redact_sensitive_text + + +def test_the_upstream_worker_context_headings_are_still_the_ones_we_cut_on(): + """Pin ``hermes_cli.kanban_db.build_worker_context``'s H2 contract. + + ``card_scope`` allow-lists CARD_SECTIONS and drops every other H2. A rename + upstream fails closed rather than leaking history into role resolution, but + it would also truncate real cards, so the coupling is asserted explicitly. + """ + kanban_db = pytest.importorskip("hermes_cli.kanban_db") + source = Path(kanban_db.__file__).read_text(encoding="utf-8") + emitted = { + match.group(1).replace("@", "").strip() + for match in re.finditer(r'lines\.append\(f?"## ([^"{]+)', source) + } + + assert emitted == {*goal.CARD_SECTIONS, *goal.HISTORY_SECTIONS} + + +def test_the_controller_evidence_heading_is_not_a_card_section(): + assert goal.CONTROLLER_EVIDENCE_HEADING.startswith("## ") + assert goal.CONTROLLER_EVIDENCE_HEADING[3:] not in goal.CARD_SECTIONS + assert goal.card_scope( + f"## Body\nreview only\n\n{goal.CONTROLLER_EVIDENCE_HEADING}\npush it" + ) == "## Body\nreview only\n\n" + + +def test_every_report_shape_round_trips_through_the_result_schema(): + for name, result in REPORTS.items(): + assert set(result) == { + "status", + "summary", + "changed_files", + "tests_run", + "artifacts", + "findings", + "blockers", + }, name + assert result["status"] in goal.RESULT_STATUSES + assert json.loads(json.dumps(result)) == result diff --git a/testing/tests/test_hermes_cli_review_corpus.py b/testing/tests/test_hermes_cli_review_corpus.py new file mode 100644 index 00000000..6ef37539 --- /dev/null +++ b/testing/tests/test_hermes_cli_review_corpus.py @@ -0,0 +1,212 @@ +"""Role resolution measured against the real Kanban card corpus. + +``data/hermes_kanban_card_corpus.jsonl`` is a verbatim snapshot of every card on +the live titan-iac, cassandra and soteria boards taken on 2026-08-17 (the metis +board was unreadable from the worker account and is absent). Each record keeps +the card's exact id, title and body plus the role this module must resolve. + +``expected_role`` was hand-labelled from each card's own deliverable, not from +the classifier's output, under one rule: a card is a *review* when it forbids +mutation and asks for a SHIP/BLOCK-shaped verdict. A read-only card whose stated +DELIVERABLE is a findings list, an observed-state table or a +certify/reject recommendation stays on the implementation regime on purpose - +the verdict is the review contract's only gate, so inferring the review role for +a card that never asked for a verdict could only fail closed. Cards that request +a push, an amend or a pull request are implementation regardless of their title. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from collections import Counter +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +DATA = Path(__file__).parent / "data" +SPEC = importlib.util.spec_from_file_location( + "cli_lane_review_corpus_test", + ROOT / "services/hermes/scripts/cli_lane_goal.py", +) +assert SPEC and SPEC.loader +goal = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = goal +SPEC.loader.exec_module(goal) + +CORPUS = [ + json.loads(line) + for line in (DATA / "hermes_kanban_card_corpus.jsonl").read_text( + encoding="utf-8" + ).splitlines() + if line.strip() +] +INCIDENT_CONTEXT = (DATA / "hermes_t_dbdcd739_context.txt").read_text(encoding="utf-8") +INCIDENT_RESULT = json.loads( + (DATA / "hermes_t_dbdcd739_result.json").read_text(encoding="utf-8") +) +CLEAN_REPORT = {"changed_files": []} + + +def _context(card: dict) -> str: + """Render one card the way ``build_worker_context`` hands it to a worker.""" + return ( + f"# Kanban task {card['id']}: {card['title']}\n\n" + "Assignee: cli-claude-xhigh\nStatus: running\n\n" + f"## Body\n{card['body']}\n" + ) + + +def _no_judge(*_args, **_kwargs): + raise AssertionError("the model judge must not be consulted for a review card") + + +@pytest.mark.parametrize( + "card", + CORPUS, + ids=[f"{card['board']}-{card['id']}" for card in CORPUS], +) +def test_every_live_card_resolves_to_its_labelled_role(card): + role, source = goal.task_role(_context(card), CLEAN_REPORT) + + assert role == card["expected_role"], card["title"] + assert source in {"inferred", "default"} + + +def test_the_corpus_covers_the_boards_and_shapes_this_change_is_about(): + counts = Counter(card["expected_role"] for card in CORPUS) + escaped = [ + card + for card in CORPUS + if "\\n" in card["body"] and "\n" not in card["body"] + ] + + assert counts[goal.REVIEW_ROLE] == 21 + assert counts[goal.IMPLEMENTATION_ROLE] == 57 + assert {card["board"] for card in CORPUS} == {"titan-iac", "cassandra", "soteria"} + # The escaped-newline shape the explicit directive contract used to miss. + assert len(escaped) == 6 + assert {card["id"] for card in escaped} >= {"t_6da029e0", "t_b04a5e67"} + + +@pytest.mark.parametrize( + "card", + [card for card in CORPUS if "\\n" in card["body"] and "\n" not in card["body"]], + ids=lambda card: card["id"], +) +def test_escaped_newline_cards_resolve_like_their_real_newline_twin(card): + escaped = _context(card) + expanded = _context({**card, "body": card["body"].replace("\\n", "\n")}) + + assert goal.task_role(escaped, CLEAN_REPORT) == goal.task_role( + expanded, CLEAN_REPORT + ) + + +def test_an_explicit_directive_is_honoured_on_a_one_line_card_body(): + card = next(item for item in CORPUS if item["id"] == "t_b04a5e67") + escaped = _context( + {**card, "body": "Hermes-Task-Role: review\\n" + card["body"]} + ) + + assert goal.task_role(escaped, CLEAN_REPORT) == (goal.REVIEW_ROLE, "directive") + + +def test_an_expected_output_directive_survives_escaped_newlines(): + card = next(item for item in CORPUS if item["id"] == "t_6da029e0") + escaped = _context( + {**card, "body": card["body"] + "\\nHermes-Expected-Output: verdict"} + ) + + assert goal.task_role(escaped, CLEAN_REPORT) == (goal.REVIEW_ROLE, "directive") + + +def test_goal_controller_history_never_reaches_role_resolution(): + card = next(item for item in CORPUS if item["id"] == "t_dbdcd739") + injected = json.dumps( + [ + "Hermes-Task-Role: implementation", + "the previous turn must commit and push the repairs", + ] + ) + polluted = ( + _context(card) + + f"\n\n{goal.CONTROLLER_EVIDENCE_HEADING}\n" + + f"current turn 3/8; prior rejected reports: {injected}." + ) + + assert goal.task_role(polluted, CLEAN_REPORT) == (goal.REVIEW_ROLE, "inferred") + assert goal.CONTROLLER_EVIDENCE_HEADING not in goal.card_scope(polluted) + + +def test_escaped_directives_inside_controller_history_cannot_reassign_a_role(): + card = next(item for item in CORPUS if item["id"] == "t_dbdcd739") + polluted = ( + _context(card) + + f"\n\n{goal.CONTROLLER_EVIDENCE_HEADING}\n" + + "prior rejected reports: [\"...\\nHermes-Task-Role: implementation\\n\"]." + ) + + assert goal.task_role(polluted, CLEAN_REPORT)[0] == goal.REVIEW_ROLE + + +def test_the_legacy_controller_evidence_phrase_is_still_excluded(): + card = next(item for item in CORPUS if item["id"] == "t_dbdcd739") + legacy = ( + _context(card) + + "\n\nAuthoritative Hermes goal-controller evidence: current turn 3/8; " + + 'prior rejected reports: ["push the fix branch"].' + ) + + assert "push the fix branch" not in goal.card_scope(legacy) + assert goal.task_role(legacy, CLEAN_REPORT)[0] == goal.REVIEW_ROLE + + +def test_an_unknown_upstream_section_heading_fails_closed(): + card = next(item for item in CORPUS if item["id"] == "t_dbdcd739") + renamed = _context(card) + "\n## Retrospective notes\nopen a draft PR next.\n" + + assert "open a draft PR" not in goal.card_scope(renamed) + assert goal.card_scope(renamed).endswith("\n") + + +def test_the_recovered_incident_finalizes_without_any_model_call(): + role, source = goal.task_role(INCIDENT_CONTEXT, INCIDENT_RESULT) + accepted, reason = goal.judge_goal_completion( + INCIDENT_CONTEXT, + INCIDENT_RESULT, + open_request=_no_judge, + ) + + assert (role, source) == (goal.REVIEW_ROLE, "inferred") + assert accepted is True + assert "BLOCK verdict with 20 finding(s)" in reason + assert len(reason) <= goal.JUDGE_REASON_LIMIT + + +def test_the_recovered_incident_context_is_trimmed_to_its_card(): + scope = goal.card_scope(INCIDENT_CONTEXT) + + assert len(scope) < len(INCIDENT_CONTEXT) + assert "## Prior attempts on this task" not in scope + assert "## Parent task results" not in scope + assert "## Recent work by" not in scope + assert "## Comment thread" not in scope + # The parent summary whose "the exact pushed SHA" phrase used to + # reclassify this card as implementation lives in the trimmed region. + assert "the exact pushed SHA" in INCIDENT_CONTEXT + assert "the exact pushed SHA" not in scope + + +def test_the_recovered_incident_is_stable_across_repeated_judging(): + verdicts = { + goal.judge_goal_completion( + INCIDENT_CONTEXT, INCIDENT_RESULT, open_request=_no_judge + ) + for _ in range(5) + } + + assert len(verdicts) == 1 diff --git a/testing/tests/test_hermes_cli_review_goal.py b/testing/tests/test_hermes_cli_review_goal.py index 05656330..788721be 100644 --- a/testing/tests/test_hermes_cli_review_goal.py +++ b/testing/tests/test_hermes_cli_review_goal.py @@ -126,17 +126,21 @@ def test_explicit_card_metadata_decides_the_task_role(role_line, expected): assert goal.task_role(card, _review())[1] == "directive" -def test_explicit_review_metadata_survives_a_changed_file_report(): +def test_explicit_review_metadata_cannot_self_certify_a_changed_file_report(): card = REVIEW_CARD.replace("## Body\n", "## Body\nHermes-Task-Role: review\n") + mutated = _review(changed_files=["services/hermes/scripts/cli_lane_goal.py"]) + assert goal.task_role(card, mutated) == (goal.IMPLEMENTATION_ROLE, "conflict") accepted, reason = goal.judge_goal_completion( card, - _review(changed_files=["reports/review.md"]), - open_request=_no_judge, + mutated, + open_request=lambda *_a, **_k: JudgeResponse("continue", "the card forbade edits"), ) - assert accepted is True - assert "resolved by directive" in reason + assert accepted is False + assert reason == "the card forbade edits" + # The declaration still holds when the report really did change nothing. + assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "directive") def test_appended_run_history_cannot_reassign_the_card_role(): @@ -297,8 +301,10 @@ def test_review_prose_about_the_reviewed_artifact_is_not_unfinished_work(): def test_implementation_prose_about_unfinished_work_is_still_rejected(): result = _implementation(summary="The broad rerun remains active before the push.") - assert goal.unfinished_result_reason(result) assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) + assert goal.judge_goal_completion( + IMPLEMENTATION_CARD, result, open_request=_no_judge + ) == (False, "completion evidence says work is unfinished: remains active") class JudgeResponse: @@ -430,7 +436,18 @@ def test_malformed_evidence_fields_are_read_without_raising(): assert goal._strings("not-a-list") == [] result = _implementation(tests_run="pytest -q", summary="Everything shipped.") - assert goal.unfinished_result_reason(result) is None + assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None + + +def test_a_blocked_implementation_report_is_rejected_without_the_read_only_guard(): + accepted, reason = goal.judge_goal_completion( + IMPLEMENTATION_CARD, + _implementation(status="blocked", summary="The remote was unreachable."), + open_request=_no_judge, + ) + + assert accepted is False + assert reason == "worker reported status 'blocked' rather than a finished task" def test_implementation_rejection_reason_carries_no_read_only_guard(): diff --git a/testing/tests/test_hermes_cli_review_roles.py b/testing/tests/test_hermes_cli_review_roles.py new file mode 100644 index 00000000..2f02cf59 --- /dev/null +++ b/testing/tests/test_hermes_cli_review_roles.py @@ -0,0 +1,159 @@ +"""Role-aware lane regressions for the goal-completion incident. + +The instrumented single-claim harness lives in +``test_hermes_cli_review_lane`` so both lane suites drive the same fake board +and the same ``execute_claim`` entry point. +""" + +from __future__ import annotations + +from test_hermes_cli_review_lane import ( + BLOCK_REVIEW, + IMPLEMENTATION_CARD, + REVIEW_CARD, + _Lane, + _no_judge, + _result, + _task, + execution, + goal, + lanes, +) + + +BARE_TOKEN_REVIEW = dict( + BLOCK_REVIEW, + summary=( + "Independent read-only review of PR #18 concludes BLOCK: the coordinator " + "drops its lease on the first tick and deployment verification is pending " + "on two of three nodes, so the change is unfit to ship." + ), +) + + +def test_a_bare_token_review_with_artifact_prose_finalizes_on_turn_one( + tmp_path, monkeypatch +): + """The end-to-end shape that burned every goal turn before the role reached + ``unfinished_result_reason``: a bare uppercase verdict token whose prose + describes the *reviewed* deployment as pending.""" + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(goal_max_turns=3), + reports=[_result(**BARE_TOKEN_REVIEW)], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert lane.rejections() == [] + assert len(lane.prompts) == 1 + assert kwargs["metadata"]["task_role"] == goal.REVIEW_ROLE + assert kwargs["metadata"]["task_role_source"] == "inferred" + assert "BLOCK verdict with 1 finding(s)" in kwargs["metadata"]["goal_judge_reason"] + + +def test_single_shot_review_completes_instead_of_discarding_its_verdict( + tmp_path, monkeypatch +): + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(goal_mode=False, goal_max_turns=1), + reports=[_result(**BARE_TOKEN_REVIEW)], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "complete" + assert kwargs["metadata"]["task_role"] == goal.REVIEW_ROLE + assert "goal_judge_reason" not in kwargs["metadata"] + + +def test_single_shot_review_without_a_verdict_fails_closed(tmp_path, monkeypatch): + """The documented single-shot change: a verdictless review is not finished.""" + monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge) + lane = _Lane( + tmp_path, + monkeypatch, + card=REVIEW_CARD, + task=_task(goal_mode=False, goal_max_turns=1), + reports=[_result(summary="Review complete; residual risks are listed.")], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "block" + assert kwargs["reason"].startswith(goal.READ_ONLY_GUARD) + assert "declares no explicit SHIP or BLOCK verdict" in kwargs["reason"] + + +def test_single_shot_implementation_still_blocks_on_unfinished_evidence( + tmp_path, monkeypatch +): + lane = _Lane( + tmp_path, + monkeypatch, + card=IMPLEMENTATION_CARD, + task=_task(goal_mode=False, goal_max_turns=1), + reports=[ + lanes.ProcessResult( + 0, + "turn one", + { + **BLOCK_REVIEW, + "summary": "The broad rerun remains active before the push.", + "changed_files": ["src/a.py"], + }, + False, + ) + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + action, kwargs = lane.terminal + assert action == "block" + assert "completion evidence says work is unfinished" in kwargs["reason"] + assert goal.READ_ONLY_GUARD not in kwargs["reason"] + + +def test_goal_controller_evidence_stays_outside_the_card(tmp_path, monkeypatch): + contexts: list[str] = [] + + def judge(objective, *_args, **_kwargs): + contexts.append(objective) + return (len(contexts) > 1, "ok" if len(contexts) > 1 else "push the branch") + + monkeypatch.setattr(goal, "judge_goal_completion", judge) + implementation = { + **BLOCK_REVIEW, + "summary": "Focused tests passed and the branch was pushed.", + "changed_files": ["src/a.py"], + "findings": [], + } + lane = _Lane( + tmp_path, + monkeypatch, + card=IMPLEMENTATION_CARD, + task=_task(goal_max_turns=2), + reports=[ + lanes.ProcessResult(0, "one", dict(implementation), False), + lanes.ProcessResult(0, "two", dict(implementation), False), + ], + ) + + execution.execute_claim("titan-iac", "t_card") + + assert lane.terminal[0] == "complete" + assert goal.CONTROLLER_EVIDENCE_HEADING in contexts[1] + assert "push the branch" in contexts[1] + assert "push the branch" not in goal.card_scope(contexts[1]) From e3ecc18d0edba8aa869f88df70a86000f3af1fa3 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:03:50 -0300 Subject: [PATCH 24/46] hermes: refresh capabilities and readiness per loop Co-Authored-By: Claude Fable 5 --- services/hermes/agent-deployment.yaml | 10 + .../hermes/scripts/cli_lane_capabilities.py | 110 ++++++++- services/hermes/scripts/cli_lane_dispatch.py | 32 ++- services/hermes/scripts/cli_lane_runner.py | 2 + testing/quality_coverage.py | 70 ++++-- testing/tests/test_hermes_agent_layout.py | 68 ++++-- testing/tests/test_hermes_cli_capabilities.py | 118 ++++++++- .../tests/test_hermes_cli_dispatch_runtime.py | 230 +++++++++++++++++- testing/tests/test_quality_contract.py | 11 +- .../tests/test_quality_coverage_helpers.py | 87 ++++++- 10 files changed, 669 insertions(+), 69 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..57611a51 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -875,6 +875,7 @@ spec: - {name: KUBECONFIG, value: /opt/data/home/.kube/config} - {name: PYTHONPATH, value: /opt/hermes} - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"} + - {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} securityContext: @@ -895,6 +896,15 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: tmp, mountPath: /tmp} + readinessProbe: + exec: + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/cli_lane_capabilities.py + initialDelaySeconds: 2 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 1 resources: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "2", memory: 6Gi} diff --git a/services/hermes/scripts/cli_lane_capabilities.py b/services/hermes/scripts/cli_lane_capabilities.py index c54c8bb0..6db205a6 100644 --- a/services/hermes/scripts/cli_lane_capabilities.py +++ b/services/hermes/scripts/cli_lane_capabilities.py @@ -1,11 +1,38 @@ #!/usr/bin/env python3 -"""Detect and publish the Kanban API boundary required by CLI lanes.""" +"""Detect and publish the Kanban API boundary required by CLI lanes. + +During a rollout the coordinator container and the cli-lane-runner worker can +run different hermes-agent image generations against one shared Kanban DB. +The image APIs that differ on the decomposition/finalization path: + +* ``complete_task``: both generations accept ``expected_run_id`` (active + exact-run completion); only the patched image accepts + ``replay_ended_run_id`` (ended-run replay). +* ``reclaim_task``: only the patched image accepts ``expected_run_id`` + (guarded orphan reclaim after a runner restart). +* ``specify_triage_task``/``decompose_triage_task``: only the patched image + accepts ``require_no_runs`` (refuses redefining tasks with run history). + +Worker-side handling per combination (proven in +testing/tests/test_hermes_cli_capabilities.py): + +* legacy worker: health/readiness stay deferred, no new claims, no unguarded + reclaim, replay-needing results stay journaled for a patched successor. +* patched worker: ready health, exact completion, guarded replay/reclaim; a + run voided by a legacy coordinator's unguarded mid-run decomposition + converges to stale/conflict evidence instead of completing. +* patched coordinator: mid-run decomposition is refused entirely, so the + active exact run completes on either worker generation. +""" from __future__ import annotations import inspect +import json +import os import sys from dataclasses import asdict, dataclass +from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable @@ -85,6 +112,7 @@ def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]: """Return a bounded operator-readable compatibility health document.""" return { "component": "cli-lane-runner", + "schema_version": 1, "state": "ready" if capabilities.ready else "deferred", "ready": capabilities.ready, "active_run_completion_safe": capabilities.exact_run_completion, @@ -117,6 +145,21 @@ def initialize_kanban_capabilities( return capabilities +def refresh_kanban_capabilities( + kanban_db: Any, + *, + health_path: Path | None = None, +) -> KanbanCapabilities: + """Reinspect the live callables and refresh bounded readiness evidence. + + ConfigMap script updates and image changes are deliberately not imported or + reloaded here. A running process observes only the already-loaded module, + which prevents a half-old/half-new Python module graph. Tests may replace + the two callables to model an image API transition at a loop boundary. + """ + return initialize_kanban_capabilities(kanban_db, health_path=health_path) + + def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities: """Return startup evidence, redetecting only when the loaded API changes.""" if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE: @@ -128,3 +171,68 @@ def runtime_health(path: Path | None = None) -> dict[str, Any]: """Read the bounded health document for probes and operator diagnostics.""" document = load_json(path or STATE_ROOT / "runtime-health.json") return document if isinstance(document, dict) else {} + + +def readiness_issue( + path: Path | None = None, + *, + now: datetime | None = None, + maximum_age_seconds: float = 60.0, +) -> str | None: + """Return a bounded readiness reason without exposing health contents.""" + destination = path or STATE_ROOT / "runtime-health.json" + try: + document = json.loads(destination.read_text(encoding="utf-8")) + except FileNotFoundError: + return "runtime health is missing" + except (OSError, UnicodeError, json.JSONDecodeError): + return "runtime health is malformed" + if not isinstance(document, dict) or document.get("schema_version") != 1: + return "runtime health is malformed" + if document.get("state") == "deferred" and document.get("ready") is False: + return "runtime compatibility is deferred" + capabilities = document.get("capabilities") + if ( + document.get("state") != "ready" + or document.get("ready") is not True + or not isinstance(capabilities, dict) + or any( + capabilities.get(name) is not True + for name in ( + "exact_run_completion", + "ended_run_replay", + "exact_run_reclaim", + ) + ) + ): + return "runtime health is malformed" + observed_at = document.get("observed_at") + if not isinstance(observed_at, str): + return "runtime health is malformed" + try: + observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + except ValueError: + return "runtime health is malformed" + if observed.tzinfo is None: + return "runtime health is malformed" + current = now or datetime.now(timezone.utc) + age = ( + current.astimezone(timezone.utc) - observed.astimezone(timezone.utc) + ).total_seconds() + if age < -5.0 or age > maximum_age_seconds: + return "runtime health is stale" + return None + + +def main() -> int: + """Provide the non-mutating Kubernetes readiness command.""" + maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60")) + issue = readiness_issue(maximum_age_seconds=maximum_age) + if issue: + print(issue, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 8a740dbf..3ed56c2f 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -10,7 +10,11 @@ import time from typing import Any from cli_lane_board import _external, _record_board_access_error, _task_value -from cli_lane_capabilities import initialize_kanban_capabilities, kanban_capabilities +from cli_lane_capabilities import ( + initialize_kanban_capabilities, + kanban_capabilities, + refresh_kanban_capabilities, +) from cli_lane_config import ( BOARD_CORRUPTION_ERRORS, DEFAULT_CLAIM_TTL, @@ -29,6 +33,7 @@ def _board_slug(board: Any) -> str: return str(board.get("slug") or board.get("id") or "") return str(getattr(board, "slug", None) or getattr(board, "id", None) or board) + def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None: """Open one board without letting localized storage faults stop other lanes.""" try: @@ -37,6 +42,7 @@ def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None: _record_board_access_error(board, error) return None + def recover_orphans() -> None: """Return external running tasks to ready after a runner/pod restart.""" from hermes_cli import kanban_db @@ -59,7 +65,10 @@ def recover_orphans() -> None: continue try: for task in kanban_db.list_tasks(conn): - if _external(task) and str(_task_value(task, "status", "")) == "running": + if ( + _external(task) + and str(_task_value(task, "status", "")) == "running" + ): task_id = str(_task_value(task, "id")) run_id = _task_value(task, "current_run_id", None) if not isinstance(run_id, int): @@ -78,6 +87,7 @@ def recover_orphans() -> None: finally: conn.close() + def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: """Atomically claim external ready tasks across all non-archived boards.""" from hermes_cli import kanban_db @@ -134,8 +144,13 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str conn.close() return claimed + def main() -> int: - """Continuously bridge external Kanban lanes to provider CLIs.""" + """Continuously bridge external Kanban lanes to provider CLIs. + + The loop is exited only by process signals or unrecoverable exceptions; + every per-board and per-claim fault is bounded inside one pass. + """ from hermes_cli import kanban_db RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -146,12 +161,20 @@ def main() -> int: futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: while True: + previous_capabilities = capabilities + capabilities = refresh_kanban_capabilities(kanban_db) + if capabilities.ready and not previous_capabilities.ready: + recover_orphans() for future in list(futures): if future.done(): try: future.result() except Exception as error: - print(f"worker future failed: {error}", file=sys.stderr, flush=True) + print( + f"worker future failed: {error}", + file=sys.stderr, + flush=True, + ) del futures[future] recover_pending_finalizations() maybe_gc_lane_artifacts() @@ -170,4 +193,3 @@ def main() -> int: future = pool.submit(execute_claim, board, task_id) futures[future] = (board, task_id) time.sleep(5) - return 0 diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index ed32e3b7..84aa185d 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -18,6 +18,8 @@ from cli_lane_capabilities import ( detect_kanban_capabilities, initialize_kanban_capabilities, kanban_capabilities, + readiness_issue, + refresh_kanban_capabilities, runtime_health, ) from cli_lane_board import ( diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..7339baa1 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,24 +3,42 @@ from __future__ import annotations import xml.etree.ElementTree as ET +import math +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Validated line and branch percentages for one source file.""" + + line: float + branch: float + + +def _percentage(class_node: ET.Element, attribute: str) -> float: + """Parse one finite Cobertura rate and fail closed outside its domain.""" + raw = class_node.attrib.get(attribute) + if raw is None: + raise ValueError(f"coverage class missing {attribute}") + value = float(raw) + if not math.isfinite(value) or value < 0.0 or value > 1.0: + raise ValueError(f"coverage class has invalid {attribute}: {raw}") + return value * 100.0 + + +def _load_rates(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load validated per-file line and branch rates from Cobertura XML.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + rates: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") - line_rate = class_node.attrib.get("line-rate") - if not filename or line_rate is None: + if not filename: continue normalized = filename.replace("\\", "/") if normalized.startswith("/"): @@ -32,8 +50,16 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 - return percentages + rates[key] = CoverageRates( + line=_percentage(class_node, "line-rate"), + branch=_percentage(class_node, "branch-rate"), + ) + return rates + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: + """Load per-file line percentages for backward-compatible callers.""" + return {path: rates.line for path, rates in _load_rates(xml_path, root).items()} def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]: @@ -45,19 +71,28 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] if not xml_path.exists(): return [f"coverage xml missing: {xml_path.relative_to(root)}"] - percentages = _load_percentages(xml_path, root) + try: + rates_by_path = _load_rates(xml_path, root) + except (ET.ParseError, OSError, UnicodeError, ValueError) as error: + return [f"coverage xml invalid: {error}"] minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = rates_by_path.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"line coverage below {minimum:.1f}%: {relative_path} " + f"({rates.line:.1f}%)" + ) + if rates.branch + 1e-9 < minimum: + issues.append( + f"branch coverage below {minimum:.1f}%: {relative_path} " + f"({rates.branch:.1f}%)" ) return issues @@ -73,7 +108,10 @@ def compute_workspace_line_coverage( if not xml_path.exists(): return 0.0 - percentages = _load_percentages(xml_path, root) + try: + percentages = _load_percentages(xml_path, root) + except (ET.ParseError, OSError, UnicodeError, ValueError): + return 0.0 samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 2e2ef65e..72ab7f05 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -36,7 +36,9 @@ def test_claude_pretool_hook_allows_normal_engineering(): assert policy.denial_reason("vault kv get kv/atlas/hermes") is None -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): +def test_claude_settings_preserve_state_and_install_three_guardrail_layers( + tmp_path: Path, +): state = tmp_path / ".claude.json" settings = tmp_path / "settings.json" state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") @@ -78,7 +80,9 @@ def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_p def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): session = tmp_path / "home/.config/herdr/session.json" session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") + session.write_text( + '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8" + ) binary = tmp_path / "tools/bin/herdr" binary.parent.mkdir(parents=True) binary.write_text("legacy", encoding="utf-8") @@ -156,9 +160,7 @@ def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): assert hostnames["operator"] == "NotIn" assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") assert hermes["resources"]["requests"]["cpu"] == "300m" @@ -223,8 +225,7 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): ) patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" + item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway" ) assert patch_init["command"][-1] == "/patched/server.py" for name in ("hermes", "terminal"): @@ -248,20 +249,29 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ "replacement" ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" + assert ( + middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] + == "http://127.0.0.1:9119" + ) ingresses = { item["metadata"]["name"]: item for item in ingress_documents if item["kind"] == "Ingress" } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" + assert ( + ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + ) + assert ( + ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-terminal-slash@kubernetescrd" + ) def test_broker_services_survive_sibling_container_readiness_loss(): @@ -322,9 +332,7 @@ def test_cli_lane_config_refresh_does_not_restart_active_work(): for item in deployment["spec"]["template"]["spec"]["containers"] if item["name"] == "cli-lane-runner" ) - mount = next( - item for item in lane["volumeMounts"] if item["name"] == "coordinator" - ) + mount = next(item for item in lane["volumeMounts"] if item["name"] == "coordinator") assert coordinator["options"]["disableNameSuffixHash"] is True assert mount == { @@ -335,12 +343,28 @@ def test_cli_lane_config_refresh_does_not_restart_active_work(): assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get( "metadata", {} ).get("annotations", {}) + assert "startupProbe" not in lane + assert "livenessProbe" not in lane + assert lane["readinessProbe"] == { + "exec": { + "command": [ + "/opt/hermes/.venv/bin/python", + "/opt/coordinator/cli_lane_capabilities.py", + ] + }, + "initialDelaySeconds": 2, + "periodSeconds": 5, + "timeoutSeconds": 2, + "failureThreshold": 1, + } + environment = {item["name"]: item["value"] for item in lane["env"]} + assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60" def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) assert "eventsRetryAttempt.current" in dockerfile assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile assert "events feed rejected (${ev.code}) — reload the page" in dockerfile diff --git a/testing/tests/test_hermes_cli_capabilities.py b/testing/tests/test_hermes_cli_capabilities.py index adc6e3ca..fd243267 100644 --- a/testing/tests/test_hermes_cli_capabilities.py +++ b/testing/tests/test_hermes_cli_capabilities.py @@ -3,14 +3,16 @@ from __future__ import annotations import json +import runpy import sys from contextlib import nullcontext +from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace import pytest -from testing.tests.test_hermes_cli_support import _completed_result, lanes +from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes def _old_complete( @@ -94,6 +96,18 @@ def test_capability_detection_requires_explicit_safety_keywords(): assert lanes._explicit_keyword(object(), "expected_run_id") is False +def test_deferred_features_name_every_absent_capability(): + """A fully legacy image defers all three safety features by exact name.""" + fully_legacy = lanes.KanbanCapabilities(False, False, False) + + assert fully_legacy.ready is False + assert fully_legacy.deferred_features == ( + "exact-run-completion", + "ended-run-replay", + "exact-run-reclaim", + ) + + def test_startup_health_moves_from_bounded_deferred_to_ready( tmp_path: Path, capsys, @@ -106,6 +120,7 @@ def test_startup_health_moves_from_bounded_deferred_to_ready( assert old.ready is False assert old_document["state"] == "deferred" + assert old_document["schema_version"] == 1 assert old_document["ready"] is False assert old_document["active_run_completion_safe"] is True assert old_document["deferred_features"] == [ @@ -123,6 +138,107 @@ def test_startup_health_moves_from_bounded_deferred_to_ready( assert lanes.runtime_health(health)["state"] == "ready" +def test_readiness_fails_closed_without_disclosing_health_values( + tmp_path: Path, + capsys, + monkeypatch, +): + """Readiness distinguishes startup, malformed, deferred, stale, and ready.""" + health = tmp_path / "runtime-health.json" + now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc) + assert lanes.readiness_issue(health, now=now) == "runtime health is missing" + + health.write_text("not-json", encoding="utf-8") + assert lanes.readiness_issue(health, now=now) == "runtime health is malformed" + + deferred = lanes._health_document(lanes.KanbanCapabilities(True, False, False)) + deferred["observed_at"] = now.isoformat() + health.write_text(json.dumps(deferred), encoding="utf-8") + assert lanes.readiness_issue(health, now=now) == "runtime compatibility is deferred" + + ready = lanes._health_document(lanes.KanbanCapabilities(True, True, True)) + ready["observed_at"] = (now - timedelta(seconds=61)).isoformat() + health.write_text(json.dumps(ready), encoding="utf-8") + assert lanes.readiness_issue(health, now=now) == "runtime health is stale" + + ready["observed_at"] = now.isoformat() + health.write_text(json.dumps(ready), encoding="utf-8") + assert lanes.readiness_issue(health, now=now) is None + + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + monkeypatch.setenv("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "31536000") + assert sys.modules["cli_lane_capabilities"].main() == 0 + assert capsys.readouterr().out == "" + + +@pytest.mark.parametrize( + "mutation", + [ + {"schema_version": 2}, + {"state": "ready", "ready": False}, + {"state": "ready", "capabilities": []}, + {"observed_at": 5}, + {"observed_at": "not-a-date"}, + {"observed_at": "2026-08-17T12:00:00"}, + ], +) +def test_readiness_rejects_inconsistent_documents( + tmp_path: Path, + mutation: dict, +): + """Inconsistent ready documents remain not-ready.""" + now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc) + document = lanes._health_document(lanes.KanbanCapabilities(True, True, True)) + document["observed_at"] = now.isoformat() + document.update(mutation) + health = tmp_path / "runtime-health.json" + health.write_text(json.dumps(document), encoding="utf-8") + + assert lanes.readiness_issue(health, now=now) == "runtime health is malformed" + + +def test_readiness_command_fails_with_only_a_bounded_reason( + tmp_path: Path, + capsys, + monkeypatch, +): + """The probe command exits nonzero and discloses nothing but the reason.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + + assert sys.modules["cli_lane_capabilities"].main() == 1 + + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.strip() == "runtime health is missing" + + +def test_capabilities_module_is_the_deployed_probe_entrypoint( + tmp_path: Path, + monkeypatch, +): + """`python cli_lane_capabilities.py` runs the readiness probe directly.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path) + + with pytest.raises(SystemExit) as excinfo: + runpy.run_path( + str(SCRIPTS / "cli_lane_capabilities.py"), + run_name="__main__", + ) + + assert excinfo.value.code == 1 + + +def test_readiness_rejects_future_health(tmp_path: Path): + """A clock-skewed future observation cannot make a stale process ready.""" + now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc) + document = lanes._health_document(lanes.KanbanCapabilities(True, True, True)) + document["observed_at"] = (now + timedelta(seconds=6)).isoformat() + health = tmp_path / "runtime-health.json" + health.write_text(json.dumps(document), encoding="utf-8") + + assert lanes.readiness_issue(health, now=now) == "runtime health is stale" + + @pytest.mark.parametrize( ("status", "run_id", "expected"), [("running", 7, "committed"), ("blocked", None, "deferred")], diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py index 74684105..06873f0a 100644 --- a/testing/tests/test_hermes_cli_dispatch_runtime.py +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations +import runpy import sys from contextlib import nullcontext from pathlib import Path @@ -9,7 +10,7 @@ from types import SimpleNamespace import pytest -from testing.tests.test_hermes_cli_support import lanes +from testing.tests.test_hermes_cli_support import SCRIPTS, lanes def test_board_slug_and_connection_failure_paths(monkeypatch): @@ -129,6 +130,28 @@ class _Pool: return self.future +class _HeldFuture: + def done(self): + return False + + +class _RecordingPool: + submissions = [] + + def __init__(self, max_workers): + self.max_workers = max_workers + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def submit(self, _function, board, task_id): + self.submissions.append((board, task_id)) + return _HeldFuture() + + def test_ready_dispatch_loop_submits_and_reaps_failed_workers( tmp_path: Path, monkeypatch, @@ -138,11 +161,9 @@ def test_ready_dispatch_loop_submits_and_reaps_failed_workers( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) - monkeypatch.setattr( - lanes, - "initialize_kanban_capabilities", - lambda _db: lanes.KanbanCapabilities(True, True, True), - ) + ready = lanes.KanbanCapabilities(True, True, True) + monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) + monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) claims = [[("cassandra", "t_loop")], []] @@ -175,11 +196,9 @@ def test_deferred_dispatch_health_never_claims_new_work( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) - monkeypatch.setattr( - lanes, - "initialize_kanban_capabilities", - lambda _db: lanes.KanbanCapabilities(True, False, False), - ) + deferred = lanes.KanbanCapabilities(True, False, False) + monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred) + monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) monkeypatch.setattr( @@ -195,3 +214,192 @@ def test_deferred_dispatch_health_never_claims_new_work( ) with pytest.raises(RuntimeError, match="stop loop"): lanes.main() + + +@pytest.mark.parametrize("start_patched", [False, True]) +def test_two_loop_api_transition_refreshes_dispatch_and_health( + tmp_path: Path, + monkeypatch, + start_patched: bool, +): + """Each loop observes legacy/patched API transitions without reloading modules.""" + + def old_complete(_conn, _task_id, *, expected_run_id=None): + return expected_run_id is not None + + def new_complete( + _conn, + _task_id, + *, + expected_run_id=None, + replay_ended_run_id=None, + ): + return expected_run_id is not None or replay_ended_run_id is not None + + def old_reclaim(_conn, _task_id, *, reason): + pytest.fail(f"unguarded legacy reclaim invoked: {reason}") + + def new_reclaim(_conn, _task_id, *, reason, expected_run_id=None): + return bool(reason and expected_run_id) + + db = SimpleNamespace( + complete_task=new_complete if start_patched else old_complete, + reclaim_task=new_reclaim if start_patched else old_reclaim, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") + recoveries = [] + monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover")) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) + claim_states = [] + + def claim(_active, _limit): + ready = lanes.kanban_capabilities(db).ready + claim_states.append(ready) + return [("cassandra", "t_loop")] if ready and not _active else [] + + monkeypatch.setattr(lanes, "claim_ready", claim) + _RecordingPool.submissions = [] + monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _RecordingPool) + journal = tmp_path / "cli-lanes/cassandra/t_loop.terminal.json" + journal.parent.mkdir(parents=True) + journal.write_text("journal", encoding="utf-8") + sleeps = [] + + def transition_then_stop(_seconds): + sleeps.append(True) + if len(sleeps) == 1: + db.complete_task = old_complete if start_patched else new_complete + db.reclaim_task = old_reclaim if start_patched else new_reclaim + return + raise RuntimeError("two loops complete") + + monkeypatch.setattr(lanes.time, "sleep", transition_then_stop) + + with pytest.raises(RuntimeError, match="two loops complete"): + lanes.main() + + assert claim_states == ([True] if start_patched else [True]) + assert _RecordingPool.submissions == ( + [("cassandra", "t_loop")] if start_patched else [("cassandra", "t_loop")] + ) + assert len(recoveries) == (1 if start_patched else 2) + assert journal.read_text(encoding="utf-8") == "journal" + expected_ready = not start_patched + assert lanes.runtime_health()["ready"] is expected_ready + + +def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks( + monkeypatch, + capsys, +): + """One faulted board or settled task never stops restart reclamation.""" + running = SimpleNamespace( + id="t_run", + status="running", + current_run_id=4, + assignee="cli-auto", + ) + settled = SimpleNamespace( + id="t_done", + status="done", + current_run_id=None, + assignee="cli-auto", + ) + reclaims = [] + + def connect(*, board): + if board == "broken": + raise OSError("volume stall") + return SimpleNamespace(close=lambda: None) + + db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "broken"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + list_tasks=lambda _conn: [settled, running], + reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append( + (task_id, kwargs["expected_run_id"]) + ), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False) + + lanes.recover_orphans() + + assert reclaims == [("t_run", 4)] + assert "temporarily skipping Kanban board 'broken'" in capsys.readouterr().err + + +def test_claim_scan_skips_foreign_lanes_and_lost_claim_races(monkeypatch): + """Foreign assignees and lost atomic claims never enter dispatch.""" + foreign = SimpleNamespace(id="t_manual", assignee="human", status="ready") + lost = SimpleNamespace(id="t_lost", assignee="cli-auto", status="ready") + won = SimpleNamespace(id="t_won", assignee="cli-auto", status="ready") + + def claim_task(_conn, task_id, **_kwargs): + return None if task_id == "t_lost" else SimpleNamespace(id=task_id) + + db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [foreign, lost, won], + claim_task=claim_task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + + assert lanes.claim_ready(set(), 3) == [("cassandra", "t_won")] + + +def test_dispatch_loop_survives_board_registry_scan_failure( + tmp_path: Path, + monkeypatch, + capsys, +): + """A failing registry scan defers claiming without stopping the loop.""" + db = SimpleNamespace() + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") + monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + ready = lanes.KanbanCapabilities(True, True, True) + monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) + monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) + + def failing_claim(_active, _limit): + raise RuntimeError("registry scan failed") + + monkeypatch.setattr(lanes, "claim_ready", failing_claim) + monkeypatch.setattr( + lanes.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")), + ) + + with pytest.raises(RuntimeError, match="stop loop"): + lanes.main() + + assert "temporarily skipping Kanban board 'board-registry'" in ( + capsys.readouterr().err + ) + + +def test_runner_module_is_the_deployed_dispatch_entrypoint(monkeypatch): + """`python cli_lane_runner.py` hands control to the dispatch loop.""" + calls = [] + monkeypatch.setattr(lanes, "main", lambda: calls.append(True) or 0) + + with pytest.raises(SystemExit) as excinfo: + runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__") + + assert excinfo.value.code == 0 + assert calls == [True] diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..59417eff 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -98,8 +98,8 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): - - + + @@ -118,7 +118,7 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): issues = run_coverage_check(contract, tmp_path, coverage_xml) - assert "coverage below 95.0%: low.py (90.0%)" in issues + assert "line coverage below 95.0%: low.py (90.0%)" in issues assert "coverage missing for tracked file: missing.py" in issues @@ -142,9 +142,8 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - - - + + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..beae7d0a 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -5,6 +5,8 @@ from __future__ import annotations import textwrap from pathlib import Path +import pytest + from testing.quality_coverage import compute_workspace_line_coverage, run_check @@ -12,10 +14,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -26,8 +33,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path - - + + @@ -41,7 +48,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -57,7 +66,7 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: - + @@ -73,4 +82,68 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: coverage_xml, ) - assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + assert issues == ["line coverage below 95.0%: relative.py (80.0%)"] + + +@pytest.mark.parametrize("branch_rate", [None, "nan", "inf", "-0.1", "1.1", "broken"]) +def test_run_check_rejects_missing_or_invalid_branch_rates( + tmp_path: Path, + branch_rate: str | None, +) -> None: + """Tracked files must carry finite Cobertura branch evidence.""" + attribute = "" if branch_rate is None else f' branch-rate="{branch_rate}"' + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + f'', + encoding="utf-8", + ) + + issues = run_check( + {"coverage": {"tracked_files": ["managed.py"]}}, + tmp_path, + coverage_xml, + ) + + assert len(issues) == 1 + assert issues[0].startswith("coverage xml invalid:") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["managed.py"]}}, + tmp_path, + coverage_xml, + ) + == 0.0 + ) + + +def test_run_check_enforces_line_and_branch_rates_independently(tmp_path: Path) -> None: + """A strong line score must not hide weak branch coverage.""" + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + '', + encoding="utf-8", + ) + + assert run_check( + {"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}}, + tmp_path, + coverage_xml, + ) == ["branch coverage below 95.0%: managed.py (50.0%)"] + + +def test_run_check_accepts_cobertura_branchless_file_semantics(tmp_path: Path) -> None: + """Coverage.py reports branchless files with branch-rate one.""" + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + '', + encoding="utf-8", + ) + + assert ( + run_check( + {"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}}, + tmp_path, + coverage_xml, + ) + == [] + ) From df75479069593415b93f3be587ae49dc63c604ce Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:03:59 -0300 Subject: [PATCH 25/46] hermes: close lane branch coverage gaps Co-Authored-By: Claude Fable 5 --- ci/scripts/semgrep_report.py | 3 +- scripts/tests/test_mailu_sync_edges.py | 96 +++++++ scripts/tests/test_mailu_sync_listener.py | 44 ++++ .../tests/test_hermes_cli_evidence_edges.py | 35 +++ .../tests/test_hermes_cli_execution_edges.py | 101 ++++++++ .../test_hermes_cli_finalization_edges.py | 65 +++++ .../test_hermes_cli_foundation_coverage.py | 47 ++++ .../tests/test_hermes_cli_goal_execution.py | 96 +++++++ testing/tests/test_hermes_cli_lane_goal.py | 11 + .../test_hermes_cli_matrix_capabilities.py | 238 ++++++++++++++++++ .../tests/test_hermes_cli_provider_edges.py | 165 ++++++++++++ .../tests/test_hermes_cli_records_edges.py | 63 +++++ .../tests/test_hermes_cli_recovery_edges.py | 146 +++++++++++ .../tests/test_hermes_cli_retention_edges.py | 64 +++++ testing/tests/test_quality_hygiene_helpers.py | 27 +- testing/tests/test_semgrep_report.py | 21 ++ 16 files changed, 1219 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/test_mailu_sync_edges.py create mode 100644 testing/tests/test_hermes_cli_matrix_capabilities.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..5175e2e6 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) + # _line_number never returns below its default, so end_line >= start_line. end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), diff --git a/scripts/tests/test_mailu_sync_edges.py b/scripts/tests/test_mailu_sync_edges.py new file mode 100644 index 00000000..db1d93d6 --- /dev/null +++ b/scripts/tests/test_mailu_sync_edges.py @@ -0,0 +1,96 @@ +"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job.""" + +from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module + +def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch): + sync = load_sync_module(monkeypatch) + monkeypatch.setattr( + sync.psycopg2, + "connect", + lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")), + ) + + assert sync.retry_request( + "request", + lambda: (_ for _ in ()).throw(AssertionError("must not run")), + attempts=0, + ) is None + assert sync.retry_db_connect(attempts=0) is None + + +def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch): + sync = load_sync_module(monkeypatch) + current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"}) + ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}}) + sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp]) + + sync.kc_update_attributes( + "token", + {"id": "u1", "username": "u1"}, + {"mailu_app_password": "abc"}, + ) + + assert sync.SESSION.put_called and sync.SESSION.get_calls == 2 + + +def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch): + sync = load_sync_module(monkeypatch) + monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}") + users = [ + { + "id": "u_disabled", + "username": "disabled", + "email": "disabled@example.com", + "enabled": False, + "attributes": {"mailu_enabled": ["true"]}, + }, + { + "id": "u_settled", + "username": "settled", + "email": "settled@example.com", + "attributes": { + "mailu_enabled": ["true"], + "mailu_email": ["settled@example.com"], + "mailu_app_password": ["keepme"], + }, + }, + ] + updated = [] + + class _Cursor: + def __init__(self): + self.executions = [] + + def execute(self, sql, params): + self.executions.append(params) + + def close(self): + return None + + class _Conn: + def __init__(self): + self.autocommit = False + self._cursor = _Cursor() + + def cursor(self, cursor_factory=None): + return self._cursor + + def close(self): + return None + + conn = _Conn() + monkeypatch.setattr(sync, "get_kc_token", lambda: "tok") + monkeypatch.setattr(sync, "kc_get_users", lambda token: users) + monkeypatch.setattr( + sync, + "kc_update_attributes", + lambda token, user, attrs: updated.append(user["id"]), + ) + monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn) + + sync.main() + + assert updated == [] + assert [params["email"] for params in conn._cursor.executions] == [ + "settled@example.com" + ] diff --git a/scripts/tests/test_mailu_sync_listener.py b/scripts/tests/test_mailu_sync_listener.py index eff7ff34..58285591 100644 --- a/scripts/tests/test_mailu_sync_listener.py +++ b/scripts/tests/test_mailu_sync_listener.py @@ -132,3 +132,47 @@ def test_listener_log_message_is_quiet(monkeypatch): handler = listener.Handler.__new__(listener.Handler) assert handler.log_message("ignored %s", "value") is None + + +def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch): + listener = load_listener_module(monkeypatch) + called = [] + monkeypatch.setattr( + listener, + "_trigger_sync_async", + lambda force=False: called.append(force) or True, + ) + handler = _handler_for(listener, "[1, 2, 3]") + + handler.do_POST() + + assert called == [False] + assert handler.responses == [202] + + +def test_listener_module_serves_forever_when_run_directly(monkeypatch): + import http.server + import runpy + + served = [] + + class _FakeServer: + def __init__(self, address, handler): + served.append((address, handler.__name__)) + + def serve_forever(self): + served.append("serving") + + monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0") + monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer) + module_path = ( + pathlib.Path(__file__).resolve().parents[2] + / "services" + / "mailu" + / "scripts" + / "mailu_sync_listener.py" + ) + + runpy.run_path(str(module_path), run_name="__main__") + + assert served == [(("", 8080), "Handler"), "serving"] diff --git a/testing/tests/test_hermes_cli_evidence_edges.py b/testing/tests/test_hermes_cli_evidence_edges.py index c28ff43f..86b02a06 100644 --- a/testing/tests/test_hermes_cli_evidence_edges.py +++ b/testing/tests/test_hermes_cli_evidence_edges.py @@ -192,3 +192,38 @@ def test_retention_failure_paths_are_bounded(tmp_path: Path, monkeypatch, capsys ) assert lanes.maybe_gc_lane_artifacts(now=1000) == 0 assert "retention deferred" in capsys.readouterr().err + + +def test_evidence_writer_rejects_a_tampered_temporary(tmp_path: Path, monkeypatch): + """A shared or replaced temporary inode never becomes durable evidence.""" + real_fstat = os.fstat + calls = [] + + def hardlinked_fstat(descriptor): + observed = real_fstat(descriptor) + calls.append(observed) + if len(calls) == 2: + return os.stat_result((*observed[:3], 2, *observed[4:])) + return observed + + monkeypatch.setattr(lanes.os, "fstat", hardlinked_fstat) + + with pytest.raises(OSError, match="not private and singly linked"): + lanes._write_json_noreplace(tmp_path / "tampered.json", {"value": "x"}) + + assert list(tmp_path.iterdir()) == [] + + +def test_evidence_writer_releases_descriptor_when_inspection_fails( + tmp_path: Path, + monkeypatch, +): + """An fstat fault closes the created descriptor without staging cleanup.""" + + def failing_fstat(_descriptor): + raise OSError("inspection failed") + + monkeypatch.setattr(lanes.os, "fstat", failing_fstat) + + with pytest.raises(OSError, match="inspection failed"): + lanes._write_json_noreplace(tmp_path / "orphaned.json", {"value": "x"}) diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py index 07b9035d..7873c86c 100644 --- a/testing/tests/test_hermes_cli_execution_edges.py +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -216,3 +216,104 @@ def test_unexpected_route_exception_blocks_the_exact_run( assert blocks[0]["expected_run_id"] == 23 assert "router down" in blocks[0]["reason"] + + +def test_capacity_fallback_without_evidence_blocks_with_provider_output( + tmp_path: Path, + monkeypatch, +): + """A fallback turn with no structured result blocks on raw provider output.""" + task = SimpleNamespace( + id="t_nofallback", + status="running", + current_run_id=22, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "attempt both providers", + add_comment=lambda *_args, **_kwargs: None, + heartbeat_worker=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + codex = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + claude = lanes.Route("claude", "fable", "high", "p", "c", "r", 1, ()) + monkeypatch.setattr( + lanes, + "select_route", + lambda _prompt, assignee, **_kwargs: claude + if assignee == "cli-claude-high" + else codex, + ) + monkeypatch.setattr(lanes, "git_handoff", lambda *_args: "handoff") + reports = [ + lanes.ProcessResult(1, "usage limit reached", None, True), + lanes.ProcessResult(1, "fallback also failed", None, False), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + + lanes.execute_claim("cassandra", "t_nofallback") + + assert reports == [] + assert blocks and "fallback also failed" in blocks[0]["reason"] + assert blocks[0]["kind"] == "capability" + + +def test_worker_blockers_become_the_block_reason_without_goal_mode( + tmp_path: Path, + monkeypatch, +): + """A structured non-completed report blocks with its own stated blockers.""" + task = SimpleNamespace( + id="t_blocked", + status="running", + current_run_id=23, + assignee="cli-auto", + max_runtime_seconds=60, + ) + blocks = [] + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda *_args: task, + worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "report the obstacle", + add_comment=lambda *_args, **_kwargs: None, + heartbeat_worker=lambda *_args, **_kwargs: True, + block_task=lambda *_args, **kwargs: blocks.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) + monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) + structured = { + **_completed_result("progress only"), + "status": "blocked", + "blockers": ["credentials expired", "remote unreachable"], + } + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(0, "out", structured, False), + ) + + lanes.execute_claim("cassandra", "t_blocked") + + assert blocks and blocks[0]["reason"] == ( + "credentials expired; remote unreachable" + ) diff --git a/testing/tests/test_hermes_cli_finalization_edges.py b/testing/tests/test_hermes_cli_finalization_edges.py index 0f2c2a2c..92b69621 100644 --- a/testing/tests/test_hermes_cli_finalization_edges.py +++ b/testing/tests/test_hermes_cli_finalization_edges.py @@ -162,3 +162,68 @@ def test_terminal_finalizer_closes_supplied_invalid_snapshots( empty = _Snapshot(None) assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid" assert empty.closed == 1 + + +def test_finalizer_rejects_invalid_names_and_unpinnable_journals( + tmp_path: Path, + monkeypatch, +): + """Invalid identities and unreadable journals return without side effects.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + (tmp_path / "cli-lanes/cassandra").mkdir(parents=True) + db = _db(None) + + foreign = tmp_path / "cli-lanes/cassandra/not-a-journal.json" + assert lanes._finalize_terminal_record(db, foreign) == "invalid" + + missing = lanes._terminal_path( + lanes.state_path("cassandra", "t_edge"), + 5, + "pending", + ) + assert lanes._finalize_terminal_record(db, missing) == "invalid" + + +class _NullSnapshot: + def __init__(self, document): + self.document = document + self.closed = 0 + + def close(self): + self.closed += 1 + + +def test_finalizer_closes_supplied_snapshots_on_every_invalid_path( + tmp_path: Path, + monkeypatch, +): + """Caller-pinned snapshots are released for named and payload rejections.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + (tmp_path / "cli-lanes/cassandra").mkdir(parents=True) + db = _db(None) + pending = lanes._terminal_path( + lanes.state_path("cassandra", "t_edge"), + 6, + "pending", + ) + + misnamed = _NullSnapshot({"kanban_state": "pending"}) + assert ( + lanes._finalize_terminal_record( + db, + tmp_path / "cli-lanes/cassandra/not-a-journal.json", + snapshot=misnamed, + ) + == "invalid" + ) + assert misnamed.closed == 1 + + empty = _NullSnapshot(None) + assert lanes._finalize_terminal_record(db, pending, snapshot=empty) == "invalid" + assert empty.closed == 1 + + malformed = _NullSnapshot({"kanban_state": "pending"}) + assert lanes._finalize_terminal_record(db, pending, snapshot=malformed) == ( + "invalid" + ) + assert malformed.closed == 1 diff --git a/testing/tests/test_hermes_cli_foundation_coverage.py b/testing/tests/test_hermes_cli_foundation_coverage.py index 71f0c44a..9184aae3 100644 --- a/testing/tests/test_hermes_cli_foundation_coverage.py +++ b/testing/tests/test_hermes_cli_foundation_coverage.py @@ -205,3 +205,50 @@ def test_switchyard_network_failure_is_explicit(): URLError("offline") ), ) + + +def test_artifact_directories_and_foreign_statuses_are_filtered( + tmp_path: Path, +): + """String directory paths and non-result statuses never become evidence.""" + workspace = tmp_path / "workspace" + (workspace / "directory").mkdir(parents=True) + + assert lanes.workspace_artifacts(workspace, ["directory"]) == [] + assert lanes._extract_json('{"status": "unknown"}') is None + + +def test_provider_events_without_sessions_or_results_stay_unparsed( + tmp_path: Path, +): + """Session persistence and result extraction require explicit fields.""" + state_file = tmp_path / "state.json" + state = {} + + assert ( + lanes._event_payload( + "codex", + json.dumps({"type": "thread.started", "thread": {}}), + state, + state_file, + ) + is None + ) + assert "codex_thread_id" not in state + assert ( + lanes._event_payload( + "codex", + json.dumps({"item": {"text": "no structured result here"}}), + state, + state_file, + ) + is None + ) + + direct = lanes._event_payload( + "codex", + json.dumps({"result": json.dumps(_completed_result("direct"))}), + state, + state_file, + ) + assert direct and direct["summary"] == "direct" diff --git a/testing/tests/test_hermes_cli_goal_execution.py b/testing/tests/test_hermes_cli_goal_execution.py index ce4ebc37..749a3d12 100644 --- a/testing/tests/test_hermes_cli_goal_execution.py +++ b/testing/tests/test_hermes_cli_goal_execution.py @@ -149,3 +149,99 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts( else: assert calls[0][1]["kind"] == "capability" assert all(connection.closed for connection in connections) + + +def test_goal_loop_recovers_from_corrupt_rejection_history( + tmp_path: Path, + monkeypatch, +): + """Corrupt durable rejection history is replaced, not trusted or fatal.""" + task = SimpleNamespace( + id="t_history", + status="running", + result=None, + current_run_id=6, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=2, + ) + calls = [] + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_history"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "achieve the goal with evidence", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + state_file = lanes.state_path("cassandra", "t_history") + lanes.atomic_json(state_file, {"goal_rejections": "corrupt-history"}) + probes = [] + monkeypatch.setattr( + lanes, + "fresh_unavailable_provider", + lambda: probes.append("health") and None, + ) + route = lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ) + monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) + reports = [ + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "done with verification", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q: passed"], + "artifacts": [], + "findings": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter([(False, "verification evidence missing"), (True, "verified")]) + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + lambda *_args, **_kwargs: next(verdicts), + ) + + lanes.execute_claim("cassandra", "t_history") + + assert reports == [] + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert len(probes) == 2 + state = lanes.load_json(state_file) + assert state["goal_rejections"] == [ + "local goal judge requested continuation: verification evidence missing" + ] diff --git a/testing/tests/test_hermes_cli_lane_goal.py b/testing/tests/test_hermes_cli_lane_goal.py index 24bea93d..0a2d7180 100644 --- a/testing/tests/test_hermes_cli_lane_goal.py +++ b/testing/tests/test_hermes_cli_lane_goal.py @@ -118,3 +118,14 @@ def test_goal_judge_fails_closed_on_invalid_response(): assert accepted is False assert "local completion judge unavailable" in reason + + +def test_completed_report_with_corrupt_test_evidence_is_still_inspected(): + """A non-list tests_run field neither crashes nor blocks the summary gate.""" + assert goal.unfinished_result_reason(_result(tests_run=None)) is None + assert goal.unfinished_result_reason( + _result( + tests_run={"suite": "still running"}, + summary="The remaining work is still running.", + ) + ) diff --git a/testing/tests/test_hermes_cli_matrix_capabilities.py b/testing/tests/test_hermes_cli_matrix_capabilities.py new file mode 100644 index 00000000..0e9498f3 --- /dev/null +++ b/testing/tests/test_hermes_cli_matrix_capabilities.py @@ -0,0 +1,238 @@ +"""Legacy/patched image compatibility matrix for mixed Hermes rollouts.""" + +from __future__ import annotations + +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_capabilities import ( + _new_reclaim, + _old_reclaim, + _terminal_document, +) +from testing.tests.test_hermes_cli_support import lanes + +def _coordinator_db(*, patched: bool): + """Model the image decomposition API used by the coordinator container.""" + if patched: + + def specify( + _conn, + _task_id, + *, + title=None, + body=None, + assignee=None, + author=None, + require_no_runs=False, + ): + return not require_no_runs + + def decompose( + _conn, + _task_id, + *, + root_assignee, + children, + author=None, + auto_promote=True, + require_no_runs=False, + ): + return None if require_no_runs else [f"{_task_id}.1"] + + else: + + def specify( + _conn, + _task_id, + *, + title=None, + body=None, + assignee=None, + author=None, + ): + return True + + def decompose( + _conn, + _task_id, + *, + root_assignee, + children, + author=None, + auto_promote=True, + ): + return [f"{_task_id}.1"] + + return SimpleNamespace( + specify_triage_task=specify, + decompose_triage_task=decompose, + ) + + +@pytest.mark.parametrize("worker_patched", [False, True]) +@pytest.mark.parametrize("coordinator_patched", [False, True]) +def test_mixed_image_rollout_compatibility_matrix( + tmp_path: Path, + monkeypatch, + coordinator_patched: bool, + worker_patched: bool, +): + """Exact legacy/patched image matrix for decomposition and finalization. + + The hermes coordinator container and the cli-lane-runner worker container + can run different image generations during a rollout while sharing one + Kanban SQLite database and these mounted scripts. The behaviors that + differ between the legacy and the patched image on this path are: + + * legacy ``complete_task`` accepts ``expected_run_id`` but not + ``replay_ended_run_id``: active exact runs complete on both images, + ended-run replay is image-gated. + * legacy ``reclaim_task`` lacks ``expected_run_id``: orphan reclaim is + image-gated (covered by test_old_image_never_calls_unguarded_reclaim). + * legacy ``specify_triage_task``/``decompose_triage_task`` lack + ``require_no_runs``: only the patched coordinator refuses to redefine + a task that already has execution history. + + Combination behavior proven here: + + * legacy coordinator + legacy worker: mid-run decomposition succeeds + unguarded; the worker defers the ended run's accepted result (journal + preserved for a patched replay) and reports deferred health. + * legacy coordinator + patched worker: mid-run decomposition succeeds + unguarded; the patched worker replays under ``replay_ended_run_id``, + the database refuses the voided run, and the result converges to + ``stale`` (conflict evidence) instead of completing a redefined task. + * patched coordinator + legacy worker: decomposition is refused while + the run exists, so the still-running exact run completes under + ``expected_run_id``; replay/reclaim stay deferred and the readiness + probe keeps the worker pod unready for new claims. + * patched coordinator + patched worker: decomposition is refused while + the run exists, the exact run completes, and the worker is ready. + """ + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + task = SimpleNamespace( + id="t_matrix", + status="running", + current_run_id=7, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + + def old_matrix_complete( + _conn, + _task_id, + *, + result=None, + summary=None, + metadata=None, + expected_run_id=None, + ): + if task.status == "running" and task.current_run_id == expected_run_id: + task.status = "done" + task.completed_run_id = expected_run_id + task.current_run_id = None + task.result = result + return True + return False + + def new_matrix_complete( + _conn, + _task_id, + *, + result=None, + summary=None, + metadata=None, + expected_run_id=None, + replay_ended_run_id=None, + ): + if replay_ended_run_id is not None: + # The decomposed run is no longer the latest ended run. + return False + return old_matrix_complete( + _conn, + _task_id, + result=result, + summary=summary, + metadata=metadata, + expected_run_id=expected_run_id, + ) + + worker_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + get_task=lambda _conn, _task_id: task, + complete_task=new_matrix_complete if worker_patched else old_matrix_complete, + reclaim_task=_new_reclaim if worker_patched else _old_reclaim, + ) + coordinator_db = _coordinator_db(patched=coordinator_patched) + + assert ( + lanes._explicit_keyword( + coordinator_db.decompose_triage_task, + "require_no_runs", + ) + is coordinator_patched + ) + assert ( + lanes._explicit_keyword( + coordinator_db.specify_triage_task, + "require_no_runs", + ) + is coordinator_patched + ) + health = tmp_path / "runtime-health.json" + capabilities = lanes.initialize_kanban_capabilities( + worker_db, + health_path=health, + ) + assert capabilities.ready is worker_patched + assert capabilities.exact_run_completion is True + if worker_patched: + assert lanes.readiness_issue(health) is None + else: + assert capabilities.deferred_features == ( + "ended-run-replay", + "exact-run-reclaim", + ) + assert ( + lanes.readiness_issue(health) == "runtime compatibility is deferred" + ) + + if coordinator_patched: + refused = coordinator_db.decompose_triage_task( + object(), + "t_matrix", + root_assignee="cli-auto", + children=[{"title": "split"}], + require_no_runs=True, + ) + assert refused is None + else: + created = coordinator_db.decompose_triage_task( + object(), + "t_matrix", + root_assignee="cli-auto", + children=[{"title": "split"}], + ) + assert created == ["t_matrix.1"] + task.status = "triage" + task.current_run_id = None + + identity = lanes.TerminalIdentity("cassandra", "t_matrix", 7, "pending") + outcome = lanes._finalize_document_db(worker_db, identity, _terminal_document()) + + if coordinator_patched: + assert outcome == "committed" + assert task.status == "done" + assert task.completed_run_id == 7 + elif worker_patched: + assert outcome == "stale" + assert task.status == "triage" + else: + assert outcome == "deferred" + assert task.status == "triage" diff --git a/testing/tests/test_hermes_cli_provider_edges.py b/testing/tests/test_hermes_cli_provider_edges.py index 21e57453..9b5b6c14 100644 --- a/testing/tests/test_hermes_cli_provider_edges.py +++ b/testing/tests/test_hermes_cli_provider_edges.py @@ -171,3 +171,168 @@ def test_codex_missing_thread_skips_colliding_restart_result( ) restart = Path(calls[1][calls[1].index("-o") + 1]) assert restart.name.endswith("provider-3.result.json") + + +def test_stream_process_trims_the_retained_line_window(tmp_path: Path, monkeypatch): + """Streamed chatter keeps only the newest bounded output window.""" + monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {}) + monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None) + read_fd, write_fd = os.pipe() + os.write(write_fd, "".join(f"{index}\n" for index in range(4100)).encode()) + os.close(write_fd) + polls = [] + + class ChatteringProcess: + pid = 4243 + returncode = 0 + stdout = os.fdopen(read_fd, "r") + + def poll(self): + polls.append(True) + return None if len(polls) < 4300 else 0 + + monkeypatch.setattr( + lanes.subprocess, + "Popen", + lambda *_args, **_kwargs: ChatteringProcess(), + ) + heartbeats = [] + + result = lanes.stream_process( + ["ignored"], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "chatter.log", + state={}, + state_file=tmp_path / "chatter-state.json", + heartbeat=lambda note: heartbeats.append(note) or True, + max_runtime=60, + ) + + assert result.returncode == 0 + assert heartbeats + published = result.output.splitlines() + assert len(published) == 4000 + assert published[0] == "100" + assert published[-1] == "4099" + + +def test_stream_process_parses_output_left_after_worker_exit( + tmp_path: Path, + monkeypatch, +): + """Buffered final output from an already-exited worker is still parsed.""" + payload = json.dumps({"result": json.dumps(_completed_result("buffered"))}) + read_fd, write_fd = os.pipe() + os.write(write_fd, f"{payload}\n".encode()) + os.close(write_fd) + fake = type( + "ExitedProcess", + (), + { + "pid": 4242, + "returncode": 0, + "poll": lambda self: 0, + "stdout": os.fdopen(read_fd, "r"), + }, + )() + monkeypatch.setattr(lanes.subprocess, "Popen", lambda *_args, **_kwargs: fake) + monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None) + + result = lanes.stream_process( + ["ignored"], + provider="codex", + cwd=tmp_path, + env={}, + log_path=tmp_path / "buffered.log", + state={}, + state_file=tmp_path / "buffered-state.json", + heartbeat=lambda _note: True, + max_runtime=60, + ) + + assert result.returncode == 0 + assert result.structured and result.structured["summary"] == "buffered" + assert payload in (tmp_path / "buffered.log").read_text(encoding="utf-8") + + +def test_descendant_snapshot_ignores_processes_that_vanish(monkeypatch): + entries = [Path("/proc/10"), Path("/proc/11"), Path("/proc/13")] + monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) + records = {10: (1, 10, 100), 11: (10, 11, 101)} + monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) + + assert lanes._descendant_processes(10) == {11: (11, 101)} + + +def test_signal_tree_targets_only_identity_verified_descendants(monkeypatch): + """Reused PIDs are never signaled; verified groups and PIDs both are.""" + records = {10: (1, 20, 100)} + monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) + group_signals = [] + pid_signals = [] + monkeypatch.setattr( + lanes.os, + "killpg", + lambda group, sig: group_signals.append((group, sig)), + ) + monkeypatch.setattr( + lanes.os, + "kill", + lambda pid, sig: pid_signals.append((pid, sig)), + ) + + lanes._signal_worker_tree( + 5, + {10: (20, 100), 11: (21, 999)}, + signal.SIGTERM, + ) + + assert sorted(group_signals) == [(5, signal.SIGTERM), (20, signal.SIGTERM)] + assert pid_signals == [(10, signal.SIGTERM)] + + +def test_claude_worker_without_any_recoverable_session_stays_unstarted( + tmp_path: Path, + monkeypatch, +): + """A session lost on resume and restart never marks the lane as started.""" + state = {"claude_started": True, "claude_session_id": "session-lost"} + state_file = tmp_path / "state.json" + commands = [] + + def stream(command, **_kwargs): + commands.append(command) + return lanes.ProcessResult( + 1, + f"{lanes.NO_CLAUDE_SESSION} session-lost", + None, + False, + ) + + monkeypatch.setattr(lanes, "stream_process", stream) + persisted = [] + monkeypatch.setattr( + sys.modules["cli_lane_provider"], + "atomic_json", + lambda path, value: persisted.append(dict(value)), + ) + route = lanes.Route("claude", "claude-fable-5", "high", "p", "c", "r", 1, ()) + + result = lanes.run_provider( + route, + "prompt", + tmp_path, + state, + state_file, + tmp_path / "worker.log", + lambda _note: True, + 60, + ) + + assert result.returncode == 1 + assert len(commands) == 2 + assert "--resume" in commands[0] + assert "--session-id" in commands[1] + assert len(persisted) == 1 diff --git a/testing/tests/test_hermes_cli_records_edges.py b/testing/tests/test_hermes_cli_records_edges.py index 3362466a..f6864918 100644 --- a/testing/tests/test_hermes_cli_records_edges.py +++ b/testing/tests/test_hermes_cli_records_edges.py @@ -228,3 +228,66 @@ def test_terminal_schema_validates_optional_identity(): record, lanes.TerminalIdentity("other", "task", 1, "pending"), ) is False + + +def test_terminal_identity_rejects_noncanonical_run_ids(): + """Identity construction fails closed before SQLite can see the run.""" + with pytest.raises(ValueError, match="positive SQLite int64"): + lanes.TerminalIdentity("cassandra", "t_identity", 0, "pending") + with pytest.raises(ValueError, match="positive SQLite int64"): + lanes.TerminalIdentity("cassandra", "t_identity", "7", "pending") + + +def test_small_snapshot_rejects_payloads_that_grow_past_the_stat_bound( + tmp_path: Path, + monkeypatch, +): + """A file appended between fstat and read never yields a snapshot.""" + path = tmp_path / "grown.json" + path.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + lanes, + "_read_bounded", + lambda _descriptor, limit: b"x" * (lanes.MAX_TERMINAL_RECORD_BYTES + 1), + ) + + assert lanes._open_small_json_snapshot(path) is None + + +def test_recovery_snapshot_marks_racing_writers_as_unstable( + tmp_path: Path, + monkeypatch, +): + """A payload that changes under the pinned read cannot be classified.""" + path = tmp_path / "unstable.json" + path.write_text('{"kanban_state": "pending"}', encoding="utf-8") + real_read = lanes._read_bounded + monkeypatch.setattr( + lanes, + "_read_bounded", + lambda descriptor, limit: real_read(descriptor, limit) + b" ", + ) + + snapshot = lanes._open_terminal_recovery_snapshot(path) + + assert snapshot is not None + assert snapshot.document is None + assert snapshot.invalid_reason == "unstable-payload" + snapshot.close() + snapshot.close() + + +def test_recovery_snapshot_releases_descriptors_on_read_faults( + tmp_path: Path, + monkeypatch, +): + """An I/O fault during the pinned read closes both held descriptors.""" + path = tmp_path / "faulted.json" + path.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + lanes, + "_read_bounded", + lambda _descriptor, _limit: (_ for _ in ()).throw(OSError("read fault")), + ) + + assert lanes._open_terminal_recovery_snapshot(path) is None diff --git a/testing/tests/test_hermes_cli_recovery_edges.py b/testing/tests/test_hermes_cli_recovery_edges.py index 27ba6773..94cd35ca 100644 --- a/testing/tests/test_hermes_cli_recovery_edges.py +++ b/testing/tests/test_hermes_cli_recovery_edges.py @@ -5,6 +5,7 @@ from __future__ import annotations import hashlib import os import sys +from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -296,3 +297,148 @@ def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts( monkeypatch.setattr(Path, "stat", denied) assert lanes._has_pending_finalization("board", "task", 12) is False + + +def test_staged_authority_rejects_a_mismatched_pathname_digest( + tmp_path: Path, + monkeypatch, +): + """A staged entry whose name does not bind its payload has no authority.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + document = _pending_terminal_record("board", "task", 5, "accepted") + board = lanes.STATE_ROOT / "board" + board.mkdir(parents=True) + forged = board / f".retire.{'f' * 16}.{'a' * 16}.0" + + assert lanes._staged_terminal_authority(forged, _Snapshot(document)) is None + + +def test_drain_staging_defers_unbounded_replacement_churn( + tmp_path: Path, + monkeypatch, +): + """A staging surface that keeps changing stops after bounded attempts.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + board = lanes.STATE_ROOT / "board" + board.mkdir(parents=True) + churning = board / ".retire.churning" + churning.write_text("0", encoding="utf-8") + + def churn(): + churning.write_text(churning.read_text(encoding="utf-8") + "x") + return 1 + + monkeypatch.setattr(lanes, "_recover_retirement_staging", churn) + + assert lanes._drain_retirement_staging() == 16 + + +def test_prepared_recovery_orders_and_bounds_disappearing_journals( + tmp_path: Path, + monkeypatch, +): + """A journal whose stat fails is ordered last instead of aborting replay.""" + path, _document = _prepared_path(tmp_path, monkeypatch) + real_stat = Path.stat + calls = [] + + def vanishing(candidate, *args, **kwargs): + if candidate == path and not calls: + calls.append("ordered") + raise OSError("stat raced") + return real_stat(candidate, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", vanishing) + monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "deferred") + + assert lanes._recover_prepared_finalizations(object()) == 0 + assert calls == ["ordered"] + assert path.exists() + + +def test_prepared_recovery_defers_a_journal_that_cannot_be_quarantined( + tmp_path: Path, + monkeypatch, +): + """An unquarantinable malformed name never spins recovery forever.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + board = lanes.STATE_ROOT / "board" + board.mkdir(parents=True) + misnamed = board / "task.run-9.terminal.prepared-zz.json" + misnamed.write_text("{}", encoding="utf-8") + attempts = [] + monkeypatch.setattr( + lanes, + "_quarantine_terminal", + lambda *_args, **_kwargs: attempts.append("quarantine"), + ) + + assert lanes._recover_prepared_finalizations(object()) == 0 + assert len(attempts) == 16 + assert misnamed.exists() + + +def test_prepared_recovery_counts_one_recovery_per_exact_run( + tmp_path: Path, + monkeypatch, +): + """Duplicate prepared evidence for one run converges to a single winner.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + identity = lanes.TerminalIdentity("board", "task", 9, "pending") + first = _pending_terminal_record("board", "task", 9, "first accepted") + second = _pending_terminal_record("board", "task", 9, "second accepted") + lanes._persist_prepared_evidence(identity, first) + lanes._persist_prepared_evidence(identity, second) + task = SimpleNamespace( + id="task", + status="running", + current_run_id=9, + completed_run_id=None, + result=None, + assignee="cli-auto", + ) + db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: SimpleNamespace(close=lambda: None), + get_task=lambda _conn, _task_id: task, + complete_task=lambda *_args, **_kwargs: True, + ) + + assert lanes._recover_prepared_finalizations(db) == 1 + conflicts = list((lanes.STATE_ROOT / "board").glob("*.terminal.conflict-*.json")) + assert len(conflicts) == 1 + + +def test_pending_guard_ignores_symlinked_boards_and_invalid_artifacts( + tmp_path: Path, + monkeypatch, +): + """Invalid journals, evidence, and staging never hold a claim open.""" + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + board = lanes.STATE_ROOT / "board" + board.mkdir(parents=True) + pending = lanes._terminal_path(lanes.state_path("board", "task"), 13) + pending.write_text("{}", encoding="utf-8") + misnamed = board / "task.run-13.terminal.prepared-zz.json" + misnamed.write_text("{}", encoding="utf-8") + invalid = board / f"task.run-13.terminal.prepared-{'a' * 32}.json" + invalid.write_text("{}", encoding="utf-8") + (board / ".retire.dangling").symlink_to(board / "missing-target") + foreign = _pending_terminal_record("board", "other", 13, "accepted") + staged, _canonical = _staged_path( + lanes.STATE_ROOT, + lanes.TerminalIdentity("board", "other", 13, "pending"), + foreign, + ) + lanes.atomic_json(staged, foreign) + + assert lanes._has_pending_finalization("board", "task", 13) is False + assert lanes._has_pending_finalization("board", "other", 13) is True + + aliased_root = tmp_path / "aliased" + aliased_root.mkdir() + (aliased_root / "linked").symlink_to(board, target_is_directory=True) + monkeypatch.setattr(lanes, "STATE_ROOT", aliased_root) + linked_pending = aliased_root / "linked" / "task.run-13.terminal.pending.json" + assert linked_pending.exists() + assert lanes._has_pending_finalization("linked", "task", 13) is False diff --git a/testing/tests/test_hermes_cli_retention_edges.py b/testing/tests/test_hermes_cli_retention_edges.py index f7deb6a2..7e56f592 100644 --- a/testing/tests/test_hermes_cli_retention_edges.py +++ b/testing/tests/test_hermes_cli_retention_edges.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from pathlib import Path from testing.tests.test_hermes_cli_support import ( @@ -87,3 +88,66 @@ def test_gc_skips_symlink_board_and_disappearing_candidate( monkeypatch.setattr(Path, "stat", disappear) assert lanes.gc_lane_artifacts() == 0 + + +def test_gc_reports_quarantine_that_left_the_entry_in_place( + tmp_path: Path, + monkeypatch, +): + """An invalid retained journal that survives quarantine is not counted.""" + root = tmp_path / "lanes" + board = root / "board" + board.mkdir(parents=True) + retained = board / "task.run-1.terminal.committed.json" + retained.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "STATE_ROOT", root) + monkeypatch.setattr(lanes, "_quarantine_terminal", lambda *_a, **_k: retained) + + assert lanes.gc_lane_artifacts() == 0 + assert retained.exists() + + +def test_gc_bounds_unpinnable_staging_and_failed_nonregular_unlinks( + tmp_path: Path, + monkeypatch, +): + """Unsnapshotable staging entries fall through to identity-safe removal.""" + root = tmp_path / "lanes" + board = root / "board" + board.mkdir(parents=True) + dangling = board / ".retire.dangling" + dangling.symlink_to(board / "missing-target") + monkeypatch.setattr(lanes, "STATE_ROOT", root) + attempts = [] + monkeypatch.setattr( + lanes, + "_unlink_artifact_if_same", + lambda path, _observed: attempts.append(path.name) and False, + ) + + assert lanes.gc_lane_artifacts() == 0 + assert attempts == [".retire.dangling"] + + +def test_gc_keeps_entries_whose_capped_unlink_loses_the_race( + tmp_path: Path, + monkeypatch, +): + """A lost inode race during count capping removes nothing else.""" + root = tmp_path / "lanes" + board = root / "board" + board.mkdir(parents=True) + newer = board / "task.candidate-2.json" + newer.write_text("{}", encoding="utf-8") + older = board / "task.candidate-1.json" + older.write_text("{}", encoding="utf-8") + os.utime(older, (1, 1)) + monkeypatch.setattr(lanes, "STATE_ROOT", root) + monkeypatch.setattr( + lanes, + "_unlink_artifact_if_same", + lambda _path, _observed: False, + ) + + assert lanes.gc_lane_artifacts(now=100.0, max_age_seconds=10**9, max_count=1) == 0 + assert older.exists() and newer.exists() diff --git a/testing/tests/test_quality_hygiene_helpers.py b/testing/tests/test_quality_hygiene_helpers.py index a303c650..9902f445 100644 --- a/testing/tests/test_quality_hygiene_helpers.py +++ b/testing/tests/test_quality_hygiene_helpers.py @@ -4,7 +4,7 @@ from __future__ import annotations from pathlib import Path -from testing.quality_hygiene import count_files_over_line_limit +from testing.quality_hygiene import count_files_over_line_limit, run_check def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None: @@ -24,3 +24,28 @@ def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> } assert count_files_over_line_limit(contract, tmp_path) == 1 + + +def test_run_check_exempts_conftest_from_naming_rules(tmp_path: Path) -> None: + """Shared conftest.py fixtures never violate test naming rules.""" + + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8") + (tests_dir / "test_named.py").write_text("checked = True\n", encoding="utf-8") + + contract = { + "hygiene": { + "max_lines": 3, + "line_limit_globs": [], + "naming_rules": [ + { + "glob": "tests/*.py", + "pattern": "^test_[a-z0-9_]+\\.py$", + "description": "pytest files use test_*.py names", + } + ], + } + } + + assert run_check(contract, tmp_path) == [] diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..89a15251 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -168,3 +168,24 @@ def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Pat payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["blocking_findings"] == 1 + + +def test_build_sonar_issues_normalizes_inverted_text_ranges() -> None: + """An end line before the start line collapses to the start line.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "path": "app/config.yaml", + "start": {"line": 9}, + "end": {"line": 4}, + } + ] + } + ) + + assert issues["issues"][0]["primaryLocation"]["textRange"] == { + "startLine": 9, + "endLine": 9, + } From 3a57371989a0f76ad5aa363455a2dc2d9561b009 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 26/46] hermes: close release-lane branch coverage gaps Cover the uncovered branches in the image release script (redirect handler, policy-read credential and size bounds, count and shape rejections, EOF manifest scan) and in the Harbor immutability helper (scope-failure verification, absent robot duration, stale-read retries, created-rule ID binding, bounded startup retries). Exclude the __main__ guard like the sibling release script. Co-Authored-By: Claude Fable 5 --- ...harbor_hermes_agent_immutability_ensure.py | 2 +- .../test_hermes_image_builder_branch_edges.py | 227 ++++++++++++++++++ 2 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 testing/tests/test_hermes_image_builder_branch_edges.py diff --git a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py index a313cc80..06475c50 100644 --- a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py +++ b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py @@ -374,5 +374,5 @@ def main() -> int: return 0 -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() raise SystemExit(main()) diff --git a/testing/tests/test_hermes_image_builder_branch_edges.py b/testing/tests/test_hermes_image_builder_branch_edges.py new file mode 100644 index 00000000..8ada5cbb --- /dev/null +++ b/testing/tests/test_hermes_image_builder_branch_edges.py @@ -0,0 +1,227 @@ +"""Branch-edge coverage for the Hermes release and Harbor policy scripts.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _publisher(module, *, immutable: bool = False, with_duration: bool = True) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + robot = { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "editable": True, + "disable": False, + "permissions": [ + {"kind": "project", "namespace": module.PROJECT, "access": access} + ], + } + if with_duration: + robot["duration"] = -1 + return robot + + +def test_release_redirect_handler_never_forwards_credentials() -> None: + """The registry opener refuses to follow any redirect target.""" + module = _load(RELEASE, "release_redirect_edges") + handler = module._NoRedirect() + assert handler.redirect_request(None, None, 0, None, None, None) is None + + +def test_release_policy_read_requires_credentials_and_bounded_body() -> None: + """The policy preflight rejects empty credentials and oversized responses.""" + module = _load(RELEASE, "release_policy_read_edges") + with pytest.raises(RuntimeError, match="credentials are empty"): + module._immutable_rules_response(username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module._immutable_rules_response( + username="robot", + password="private", + opener=lambda *_args: Response(200, b"x" * 1_048_577), + ) + + +@pytest.mark.parametrize("headers", [{}, {"X-Total-Count": "many"}]) +def test_release_policy_rejects_missing_or_invalid_count(headers: dict) -> None: + """A proxy that strips or mangles count evidence cannot prove completeness.""" + module = _load(RELEASE, "release_policy_count_edges") + with pytest.raises(RuntimeError, match="total count"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, evidence=headers: Response(200, b"[]", evidence), + ) + + +@pytest.mark.parametrize("body", [b"{}", b"[1]"]) +def test_release_policy_rejects_non_rule_list_shapes(body: bytes) -> None: + """Valid JSON that is not a list of rule objects fails the preflight.""" + module = _load(RELEASE, "release_policy_shape_edges") + with pytest.raises(RuntimeError, match="invalid shape"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, value=body: Response(200, value, _count(1)), + ) + + +def test_renderer_ignores_trailing_entry_without_digest_at_eof() -> None: + """A dangling final image entry cannot shadow the one complete digest.""" + module = _load(RELEASE, "release_render_eof_edges") + digest = "sha256:" + "6" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: dangling\n" + ) + rendered = module.render_kustomization(source, digest) + assert f"digest: {digest}\n" in rendered + assert rendered.endswith("newTag: dangling\n") + + +def test_harbor_verified_publisher_rejects_exact_identity_with_bad_scope() -> None: + """An exact robot identity still fails verification on a malformed scope.""" + module = _load(HARBOR, "harbor_verified_scope_edges") + robot = _publisher(module) + robot["permissions"] = "bad" + assert module._verified_publisher(robot) is False + + +def test_harbor_publisher_update_omits_absent_duration(monkeypatch) -> None: + """A robot without a configured duration is preserved without inventing one.""" + module = _load(HARBOR, "harbor_duration_edges") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module, with_duration=False)).encode(), {}), + (200, b"", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "duration" not in payload + + +def test_harbor_publisher_verify_retries_past_invalid_json(monkeypatch) -> None: + """One stale unreadable reread retries instead of passing or failing outright.""" + module = _load(HARBOR, "harbor_verify_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module)).encode(), {}), + (200, b"", {}), + (200, b"not-json", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert sleeps == [1] + + +def test_harbor_rule_verify_requires_the_created_rule_id(monkeypatch) -> None: + """A matching rule under a different ID is a stale read, not creation proof.""" + module = _load(HARBOR, "harbor_rule_id_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + location = "/api/v2.0/projects/bstein/immutabletagrules/23" + client = FakeClient( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": location}), + (200, json.dumps([{"id": 99, **module.EXPECTED_RULE}]).encode(), _count(1)), + (200, json.dumps([{"id": 23, **module.EXPECTED_RULE}]).encode(), _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert sleeps == [1] + + +def test_harbor_main_reraises_after_bounded_unavailable_retries( + monkeypatch, tmp_path: Path +) -> None: + """Persistent Harbor unavailability surfaces instead of looping forever.""" + module = _load(HARBOR, "harbor_main_retry_edges") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + + def never_ready(_client): + raise module.HarborUnavailable("still warming") + + monkeypatch.setattr(module, "ensure_rule", never_ready) + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + with pytest.raises(module.HarborUnavailable, match="still warming"): + module.main() + assert len(sleeps) == 11 From 64272f52d2e6f2eef728c38432722a249777574b Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 27/46] hermes: lift remaining tracked modules to the branch floor Exercise the mailu sync retry, attribute, and skip branches, the listener non-object JSON path, and the hygiene conftest skip; drop the unreachable inverted-range clamp in the semgrep report (the line helper already floors the end line) and pin that behavior with a test. Exclude the mailu __main__ guards from measurement. Co-Authored-By: Claude Fable 5 --- ci/scripts/semgrep_report.py | 3 +- scripts/tests/test_mailu_sync.py | 77 +++++++++++++++++++ scripts/tests/test_mailu_sync_listener.py | 12 +++ services/mailu/scripts/mailu_sync.py | 2 +- services/mailu/scripts/mailu_sync_listener.py | 2 +- testing/tests/test_quality_hygiene_helpers.py | 28 ++++++- testing/tests/test_semgrep_report.py | 23 ++++++ 7 files changed, 142 insertions(+), 5 deletions(-) diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..80fb5953 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) + # _line_number floors the end line at start_line, so the range never inverts. end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), diff --git a/scripts/tests/test_mailu_sync.py b/scripts/tests/test_mailu_sync.py index 4d6207fe..c5a35954 100644 --- a/scripts/tests/test_mailu_sync.py +++ b/scripts/tests/test_mailu_sync.py @@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch): sync.retry_db_connect(attempts=1) +def test_retry_helpers_return_none_without_attempts(monkeypatch): + sync = load_sync_module(monkeypatch) + + assert sync.retry_request("request", lambda: "never", attempts=0) is None + assert sync.retry_db_connect(attempts=0) is None + + +def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch): + sync = load_sync_module(monkeypatch) + current_resp = _FakeResponse({"attributes": "not-a-dict"}) + ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}}) + sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp]) + + sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"}) + + assert sync.SESSION.put_called + + def test_ensure_mailu_user_skips_foreign_domain(monkeypatch): sync = load_sync_module(monkeypatch) executed = [] @@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch): # Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled. assert len(updated) == 3 assert conns and len(conns[0]._cursor.executions) == 3 + + +def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch): + sync = load_sync_module(monkeypatch) + monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}") + users = [ + { + "id": "u1", + "username": "disabled", + "email": "disabled@example.com", + "enabled": False, + "attributes": {"mailu_enabled": ["true"]}, + }, + { + "id": "u2", + "username": "settled", + "email": "settled@example.com", + "attributes": { + "mailu_enabled": ["true"], + "mailu_email": ["settled@example.com"], + "mailu_app_password": ["existing"], + }, + }, + ] + updated = [] + + class _Cursor: + def __init__(self): + self.executions = [] + + def execute(self, sql, params): + self.executions.append(params) + + def close(self): + return None + + class _Conn: + def __init__(self): + self.autocommit = False + self._cursor = _Cursor() + + def cursor(self, cursor_factory=None): + return self._cursor + + def close(self): + return None + + conn = _Conn() + monkeypatch.setattr(sync, "get_kc_token", lambda: "tok") + monkeypatch.setattr(sync, "kc_get_users", lambda token: users) + monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"])) + monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn) + + sync.main() + + # The already-provisioned user is synced without a Keycloak write; the + # disabled user never reaches the mailbox upsert. + assert updated == [] + assert len(conn._cursor.executions) == 1 diff --git a/scripts/tests/test_mailu_sync_listener.py b/scripts/tests/test_mailu_sync_listener.py index eff7ff34..d9d99229 100644 --- a/scripts/tests/test_mailu_sync_listener.py +++ b/scripts/tests/test_mailu_sync_listener.py @@ -127,6 +127,18 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch): assert handler.responses == [200] +def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch): + listener = load_listener_module(monkeypatch) + called = [] + monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True) + handler = _handler_for(listener, "[1, 2]") + + handler.do_POST() + + assert called == [False] + assert handler.responses == [202] + + def test_listener_log_message_is_quiet(monkeypatch): listener = load_listener_module(monkeypatch) handler = listener.Handler.__new__(listener.Handler) diff --git a/services/mailu/scripts/mailu_sync.py b/services/mailu/scripts/mailu_sync.py index d04ee25a..5b8837a3 100644 --- a/services/mailu/scripts/mailu_sync.py +++ b/services/mailu/scripts/mailu_sync.py @@ -314,7 +314,7 @@ def main(): conn.close() -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() try: main() except Exception as exc: diff --git a/services/mailu/scripts/mailu_sync_listener.py b/services/mailu/scripts/mailu_sync_listener.py index 48f3d4fe..63c9824f 100644 --- a/services/mailu/scripts/mailu_sync_listener.py +++ b/services/mailu/scripts/mailu_sync_listener.py @@ -100,6 +100,6 @@ class Handler(http.server.BaseHTTPRequestHandler): return -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - server loop server = http.server.ThreadingHTTPServer(("", 8080), Handler) server.serve_forever() diff --git a/testing/tests/test_quality_hygiene_helpers.py b/testing/tests/test_quality_hygiene_helpers.py index a303c650..9a96c86f 100644 --- a/testing/tests/test_quality_hygiene_helpers.py +++ b/testing/tests/test_quality_hygiene_helpers.py @@ -4,7 +4,33 @@ from __future__ import annotations from pathlib import Path -from testing.quality_hygiene import count_files_over_line_limit +from testing.quality_hygiene import count_files_over_line_limit, run_check + + +def test_run_check_skips_conftest_files_in_naming_rules(tmp_path: Path) -> None: + """A conftest.py is pytest plumbing, never a naming-rule violation.""" + + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8") + (tests_dir / "helper.py").write_text("value = 1\n", encoding="utf-8") + + contract = { + "hygiene": { + "naming_rules": [ + { + "glob": "tests/*.py", + "pattern": "^test_[a-z0-9_]+\\.py$", + "description": "Pytest files use test_*.py names.", + } + ] + } + } + + issues = run_check(contract, tmp_path) + + assert len(issues) == 1 + assert "helper.py" in issues[0] def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None: diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..7fc62346 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,29 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_build_sonar_issues_clamps_inverted_line_ranges() -> None: + """An end line before the start line should collapse to a valid range.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "check_id": "python.lang.correctness.range", + "path": "app/main.py", + "start": {"line": 9}, + "end": {"line": 3}, + "extra": {"severity": "WARNING", "message": "inverted"}, + } + ] + } + ) + + assert issues["issues"][0]["primaryLocation"]["textRange"] == { + "startLine": 9, + "endLine": 9, + } + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" From f77119b23855f8d90559bc1bbf42bae8d8509d29 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 28/46] hermes: enforce per-file branch coverage in the quality gate The coverage check read only line-rate, so a file could pass with weak branch coverage. Load both Cobertura rates and fail any tracked file below the 95% floor on either metric, failing closed when branch evidence is absent. Prove enforcement end to end with a synthetic fully-lined but branch-weak file failing run_profile. Co-Authored-By: Claude Fable 5 --- testing/quality_coverage.py | 47 +++++++++----- testing/quality_gate.py | 2 +- testing/tests/test_quality_contract.py | 14 ++-- .../tests/test_quality_coverage_helpers.py | 65 ++++++++++++++++++- testing/tests/test_quality_gate.py | 26 ++++++++ 5 files changed, 127 insertions(+), 27 deletions(-) diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..d99aea03 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +def _load_rates(xml_path: Path, root: Path) -> dict[str, dict[str, float | None]]: + """Load per-file line and branch percentages from a Cobertura XML report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ @@ -16,7 +16,7 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + rates: dict[str, dict[str, float | None]] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") @@ -32,33 +32,44 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 - return percentages + branch_rate = class_node.attrib.get("branch-rate") + rates[key] = { + "line": float(line_rate) * 100.0, + "branch": None if branch_rate is None else float(branch_rate) * 100.0, + } + return rates def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]: """Return human-readable issues for tracked files below the coverage floor. - The report is intentionally per-file so a single weak module cannot hide - behind aggregate suite coverage. + Line and branch rates are both enforced per file so a single weak module + cannot hide behind aggregate suite coverage or line-only reporting. """ if not xml_path.exists(): return [f"coverage xml missing: {xml_path.relative_to(root)}"] - percentages = _load_percentages(xml_path, root) + rates = _load_rates(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + file_rates = rates.get(normalized) + if file_rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: - issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" - ) + for metric in ("line", "branch"): + percent = file_rates[metric] + if percent is None: + issues.append( + f"{metric} coverage missing for tracked file: {relative_path}" + ) + elif percent + 1e-9 < minimum: + issues.append( + f"{metric} coverage below {minimum:.1f}%: " + f"{relative_path} ({percent:.1f}%)" + ) return issues @@ -73,13 +84,13 @@ def compute_workspace_line_coverage( if not xml_path.exists(): return 0.0 - percentages = _load_percentages(xml_path, root) + rates = _load_rates(xml_path, root) samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + file_rates = rates.get(normalized) + if file_rates is not None: + samples.append(file_rates["line"]) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_gate.py b/testing/quality_gate.py index dc5dd1e4..d436281c 100644 --- a/testing/quality_gate.py +++ b/testing/quality_gate.py @@ -332,7 +332,7 @@ def run_profile( results.append( _result( "coverage", - "Per-file 95% coverage floor for tracked quality-managed modules.", + "Per-file 95% line and branch coverage floor for tracked quality-managed modules.", _status_from_issues(issues), issues=issues, coverage_xml=str(coverage_xml.relative_to(root)), diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..f7bf38b7 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -98,8 +98,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): - - + + + @@ -112,13 +113,14 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): contract = { "coverage": { "minimum_percent": 95.0, - "tracked_files": ["ok.py", "low.py", "missing.py"], + "tracked_files": ["ok.py", "low.py", "weak_branches.py", "missing.py"], } } issues = run_coverage_check(contract, tmp_path, coverage_xml) - assert "coverage below 95.0%: low.py (90.0%)" in issues + assert "line coverage below 95.0%: low.py (90.0%)" in issues + assert "branch coverage below 95.0%: weak_branches.py (90.0%)" in issues assert "coverage missing for tracked file: missing.py" in issues @@ -142,8 +144,8 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - - + + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..7e6d90f3 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -57,7 +57,7 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: - + @@ -73,4 +73,65 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: coverage_xml, ) - assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + assert issues == ["line coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None: + """Full line coverage must never hide a file whose branch coverage is weak.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py", "solid.py"]}}, + tmp_path, + coverage_xml, + ) + + assert issues == ["branch coverage below 95.0%: weak.py (90.0%)"] + + +def test_run_check_fails_closed_when_branch_evidence_is_absent(tmp_path: Path) -> None: + """A report generated without branch measurement cannot satisfy the gate.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["unmeasured.py"]}}, + tmp_path, + coverage_xml, + ) + + assert issues == ["branch coverage missing for tracked file: unmeasured.py"] diff --git a/testing/tests/test_quality_gate.py b/testing/tests/test_quality_gate.py index 256a78e6..fd334d85 100644 --- a/testing/tests/test_quality_gate.py +++ b/testing/tests/test_quality_gate.py @@ -60,6 +60,32 @@ def test_run_profile_aggregates_internal_and_pytest_results(tmp_path: Path, monk assert any(result.get("junit") == "build/junit-unit.xml" for result in summary["results"]) +def test_run_profile_fails_synthetic_file_below_branch_floor(tmp_path: Path): + """The gate itself must fail a tracked file with full lines but weak branches.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + (build_dir / "coverage-unit.xml").write_text( + "" + '' + "", + encoding="utf-8", + ) + + contract = { + "profiles": {"local": ["coverage"]}, + "pytest_suites": {"unit": {"coverage_xml": "build/coverage-unit.xml"}}, + "coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py"]}, + } + + summary = quality_gate.run_profile(contract, tmp_path, "local", build_dir) + + assert summary["status"] == "failed" + coverage_result = summary["results"][0] + assert coverage_result["name"] == "coverage" + assert coverage_result["issues"] == ["branch coverage below 95.0%: weak.py (90.0%)"] + assert summary["workspace_line_coverage_percent"] == 100.0 + + def test_main_writes_summary_file(tmp_path: Path, monkeypatch): summary = {"status": "ok", "profile": "local", "results": [], "manual_scripts": []} monkeypatch.chdir(tmp_path) From 2019b082760b2b2840243c7dcdc124e7adc0601e Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:35 -0300 Subject: [PATCH 29/46] gitea: require literal branch protection rules Replace the reimplemented gobwas/glob matcher with exact-literal rule matching that fails closed on any special or malformed pattern, accepts legacy zero priorities, and rejects duplicate primary-branch rules. Co-Authored-By: Claude Fable 5 --- .../scripts/gitea_branch_protection_check.py | 111 ++++-------------- .../test_hermes_gitea_branch_coverage.py | 70 +++++++---- .../tests/test_hermes_scm_broker_policy.py | 86 ++++---------- 3 files changed, 98 insertions(+), 169 deletions(-) diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py index e3f112c9..5cb9717d 100644 --- a/services/gitea/scripts/gitea_branch_protection_check.py +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -7,7 +7,6 @@ import argparse import datetime import json import os -import re import stat from pathlib import Path @@ -35,90 +34,25 @@ def _read_bounded(path: Path) -> bytes: return value -def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]: - """Compile the gobwas/glob syntax Gitea v1.23.8 uses.""" - pieces: list[str] = [] - while position < len(pattern): - character = pattern[position] - if character in terminators: - break - if character == "\\": - position += 1 - if position >= len(pattern): - raise ValueError("trailing escape") - pieces.append(re.escape(pattern[position])) - elif character == "*": - if position + 1 < len(pattern) and pattern[position + 1] == "*": - pieces.append(".*") - position += 1 - else: - pieces.append("[^/]*") - elif character == "?": - pieces.append("[^/]") - elif character == "[": - end = pattern.find("]", position + 1) - if end < 0: - raise ValueError("unterminated range") - value = pattern[position + 1 : end] - if not value: - raise ValueError("empty range") - negate = value.startswith("!") - value = value[1:] if negate else value - if not value or "[" in value or "\\" in value: - raise ValueError("invalid range") - if "-" in value: - if value.count("-") != 1 or value.startswith("-") or value.endswith("-"): - raise ValueError("invalid range") - low, high = value.split("-") - if len(low) != 1 or len(high) != 1 or ord(high) < ord(low): - raise ValueError("invalid range") - content = re.escape(low) + "-" + re.escape(high) - else: - content = re.escape(value) - pieces.append("[" + ("^" if negate else "") + content + "]") - position = end - elif character == "{": - alternatives: list[str] = [] - position += 1 - while True: - alternative, position = _glob_regex(pattern, position, ",}") - alternatives.append(alternative) - if position >= len(pattern): - raise ValueError("unterminated alternatives") - if pattern[position] == "}": - break - position += 1 - if len(alternatives) < 2 or any(not item for item in alternatives): - raise ValueError("invalid alternatives") - pieces.append("(?:" + "|".join(alternatives) + ")") - elif character in "]}": - # A closing delimiter outside its grammar context is literal in - # gobwas/glob's lexer. - pieces.append(re.escape(character)) - else: - pieces.append(re.escape(character)) - position += 1 - return "".join(pieces), position - - def _is_plain(pattern: str) -> bool: return not any(character in SPECIAL for character in pattern) -def _matches(pattern: str, branch: str) -> bool: +def _literal_rule(pattern: str) -> str: + """Return one exact rule name or reject Gitea glob interpretation. + + Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that + grammar and its error quoting rules here would create a second policy + engine. The reconciler instead accepts only plain ASCII literals. This + deliberately rejects braces, ranges, escapes, malformed globs, and even + special rules that appear unrelated: an operator must remove ambiguity + before this job may attest the primary-branch boundary. + """ if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: raise PolicyError("branch protection rule name is invalid") - if _is_plain(pattern): - return pattern.casefold() == branch.casefold() - try: - expression, position = _glob_regex(pattern) - if position != len(pattern): - raise ValueError("incomplete glob") - except (re.error, ValueError): - # Gitea quotes an invalid special pattern and matches it literally. - # A literal containing a special byte cannot equal main or master. - return False - return re.fullmatch(expression, branch) is not None + if not _is_plain(pattern): + raise PolicyError("glob branch protection rules require human review") + return pattern def _created(value: object) -> datetime.datetime: @@ -159,7 +93,7 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str: data = json.loads(value) if not isinstance(data, list) or len(data) > MAX_RULES: raise PolicyError("branch protection response is invalid") - ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = [] + ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = [] for index, raw in enumerate(data): if not isinstance(raw, dict): raise PolicyError("branch protection entry is invalid") @@ -168,18 +102,23 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str: if ( not isinstance(priority, int) or isinstance(priority, bool) - or not 1 <= priority <= 1_000_000 + or not 0 <= priority <= 1_000_000 or not isinstance(rule_name, str) ): raise PolicyError("branch protection priority is ambiguous") created = _created(raw.get("created_at")) - if _matches(rule_name, branch): - # Gitea v1.23.8 sorts by priority, then puts plain names before - # globs, then uses creation time. The API exposes every component. - ordered.append((priority, not _is_plain(rule_name), created, index, raw)) + literal = _literal_rule(rule_name) + if literal == branch: + # With every rule constrained to a plain literal, Gitea's glob + # comparator cannot affect this result. Preserve API ordering: + # priority first (including legacy zero), creation time, then the + # stable response order for an otherwise exact tie. + ordered.append((priority, created, index, raw)) if not ordered: return "ABSENT" - effective = min(ordered)[4] + if len(ordered) != 1: + raise PolicyError("duplicate primary-branch protections are ambiguous") + effective = ordered[0][3] for key, expected in _required(reviewer).items(): if effective.get(key) != expected: raise PolicyError( diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py index 362e8726..48421f01 100644 --- a/testing/tests/test_hermes_gitea_branch_coverage.py +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -83,27 +83,16 @@ def test_read_bounded_rejects_unsafe_or_changed_input( monkeypatch.setattr(module.os, "read", real_read) -@pytest.mark.parametrize( - ("pattern", "branch", "expected"), - [ - ("m?in", "main", True), - ("m[ai]in", "main", True), - ("m[!z]in", "main", True), - ("m]ain", "m]ain", True), - ("m}ain", "m}ain", True), - ("m[a-z]in", "main", True), - ("m{ain,aster}", "main", True), - ("m\\ain", "main", True), - ], -) -def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): - module = _module("branch_glob_supported") - assert module._matches(pattern, branch) is expected - - @pytest.mark.parametrize( "pattern", [ + "{main}", + "{main,master}", + "m?in", + "m[ai]in", + "m[!z]in", + "m[a-z]in", + "m\\ain", "m\\", "m[", "m[]", @@ -120,16 +109,26 @@ def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): "m{ain,aster", ], ) -def test_invalid_globs_are_literal_nonmatches(pattern): +def test_every_special_or_malformed_glob_fails_closed(pattern): module = _module("branch_glob_invalid") - assert module._matches(pattern, "main") is False + with pytest.raises(module.PolicyError, match="glob branch protection"): + module._literal_rule(pattern) @pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) def test_rule_names_are_ascii_and_bounded(pattern): module = _module("branch_name_bounds") with pytest.raises(module.PolicyError, match="rule name"): - module._matches(pattern, "main") + module._literal_rule(pattern) + + +def test_plain_rule_matching_is_exact_and_case_sensitive(): + module = _module("branch_literal_case") + assert module._literal_rule("main") == "main" + assert module._literal_rule("MAIN") == "MAIN" + assert module.evaluate( + json.dumps([_rule(module, name="MAIN")]).encode(), "main", "bstein" + ) == "ABSENT" @pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) @@ -157,7 +156,7 @@ def test_evaluate_rejects_unknown_branch_and_excess_rules(): module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") -@pytest.mark.parametrize("priority", [True, 0, 1_000_001, "1"]) +@pytest.mark.parametrize("priority", [True, -1, 1_000_001, "1"]) def test_evaluate_rejects_ambiguous_priority_types(priority): module = _module("branch_priority_bounds") rule = _rule(module, priority=priority) @@ -165,6 +164,33 @@ def test_evaluate_rejects_ambiguous_priority_types(priority): module.evaluate(json.dumps([rule]).encode(), "main", "bstein") +def test_legacy_zero_priority_is_accepted_but_duplicate_ties_fail_closed(): + module = _module("branch_priority_zero") + assert module.evaluate( + json.dumps([_rule(module, priority=0)]).encode(), "main", "bstein" + ) == "PRESENT" + + first = _rule(module, priority=0, created_at="2026-01-01T00:00:00Z") + second = _rule( + module, + priority=0, + created_at="2026-01-01T00:00:00Z", + enable_force_push=True, + ) + with pytest.raises(module.PolicyError, match="duplicate primary-branch"): + module.evaluate(json.dumps([first, second]).encode(), "main", "bstein") + + +def test_any_special_rule_blocks_absent_create_and_present_readback(): + module = _module("branch_special_readback") + for rules in ( + [_rule(module, name="feature/*")], + [_rule(module), _rule(module, name="{main}", priority=0)], + ): + with pytest.raises(module.PolicyError, match="human review"): + module.evaluate(json.dumps(rules).encode(), "main", "bstein") + + def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): module = _module("branch_main_paths") source = tmp_path / "rules.json" diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py index 25d94fa0..163cbc91 100644 --- a/testing/tests/test_hermes_scm_broker_policy.py +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -216,83 +216,47 @@ def _protection(rule_name: str, priority: int, helper, **overrides): return value -def test_branch_protection_uses_first_effective_glob_by_priority(): +def test_branch_protection_requires_exact_literal_rules_only(): helper = _load_path( "branch_protection_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - earlier_drift = _protection("m*", 1, helper, required_approvals=0) - later_exact = _protection("main", 2, helper) + exact = _protection("main", 2, helper) + assert ( + helper.evaluate(json.dumps([exact]).encode(), "main", "bstein") == "PRESENT" + ) + drifted = _protection("main", 2, helper, required_approvals=0) with pytest.raises(helper.PolicyError, match="effective main protection differs"): - helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + helper.evaluate(json.dumps([drifted]).encode(), "main", "bstein") - earlier_drift["required_approvals"] = 1 - assert ( - helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") - == "PRESENT" - ) - brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0) - with pytest.raises(helper.PolicyError, match="effective master protection differs"): - helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein") - - upper_glob = _protection("M*", 1, helper, required_approvals=0) - assert ( - helper.evaluate(json.dumps([upper_glob, later_exact]).encode(), "main", "bstein") - == "PRESENT" - ) - - -def test_branch_protection_matches_gitea_v1238_tie_order(): - helper = _load_path( - "branch_protection_order_test", - ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", - ) - older_glob = _protection( - "m*", 1, helper, created_at="2025-01-01T00:00:00Z", required_approvals=0 - ) - newer_plain = _protection( - "MAIN", 1, helper, created_at="2026-01-01T00:00:00Z" - ) - assert ( - helper.evaluate(json.dumps([older_glob, newer_plain]).encode(), "main", "bstein") - == "PRESENT" - ) - - older_glob["created_at"] = "2026-01-02T00:00:00Z" - earlier_glob = _protection( - "m{ain,aster}", - 1, - helper, - created_at="2026-01-01T00:00:00Z", - required_approvals=0, - ) - with pytest.raises(helper.PolicyError, match="effective main protection differs"): - helper.evaluate(json.dumps([older_glob, earlier_glob]).encode(), "main", "bstein") + duplicate = _protection("main", 1, helper) + with pytest.raises(helper.PolicyError, match="duplicate primary-branch"): + helper.evaluate(json.dumps([exact, duplicate]).encode(), "main", "bstein") @pytest.mark.parametrize( - ("rule_name", "branch", "expected"), + "rule_name", [ - ("release/*", "release/v1.17", True), - ("release/**/v1.17", "release/test/1/v1.17", True), - ("release/*/v1.17", "release/test/1/v1.17", False), - ("*", "release/v1.16", False), - ("**", "release/v1.16", True), - ("MAIN", "main", True), - ("M*", "main", False), - ("m{ain,aster}", "master", True), - (r"m\ain", "main", True), + "release/*", + "release/**/v1.17", + "*", + "**", + "M*", + "m{ain,aster}", + r"m\ain", + "m?in", + "m[ai]in", ], ) -def test_branch_glob_matches_gitea_v1238_source_corpus( - rule_name: str, branch: str, expected: bool -): +def test_branch_glob_rules_fail_closed_pending_human_review(rule_name: str): helper = _load_path( "branch_protection_glob_parity_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - assert helper._matches(rule_name, branch) is expected + rules = [_protection(rule_name, 1, helper)] + with pytest.raises(helper.PolicyError, match="human review"): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") @pytest.mark.parametrize( @@ -317,5 +281,5 @@ def test_branch_protection_reports_absent_only_when_no_rule_matches(): "branch_protection_absent_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - rules = [_protection("release/*", 1, helper)] + rules = [_protection("develop", 1, helper)] assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" From 578239a49651f4e9ba5bb348fc7a78a3f11e5f30 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:35 -0300 Subject: [PATCH 30/46] hermes: bound SCM control calls with hard deadlines Run every Gitea API and broker control exchange inside a killable helper process whose connect, send, and read share one absolute wall-clock deadline, and add a watchdog that force-closes streaming connections at expiry. Redirects are rejected before authentication headers can move. Co-Authored-By: Claude Fable 5 --- .../scm-common/scripts/deadline_http.py | 302 +++++++++++++++++ .../hermes/scm-common/scripts/gitea_api.py | 22 +- .../scm-common/scripts/scm_broker_client.py | 20 +- testing/tests/test_hermes_deadline_http.py | 309 ++++++++++++++++++ .../test_hermes_gitea_internal_coverage.py | 15 +- testing/tests/test_hermes_gitea_pr_client.py | 19 +- .../test_hermes_scm_broker_client_coverage.py | 17 +- 7 files changed, 663 insertions(+), 41 deletions(-) create mode 100644 services/hermes/scm-common/scripts/deadline_http.py create mode 100644 testing/tests/test_hermes_deadline_http.py diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py new file mode 100644 index 00000000..a19abccc --- /dev/null +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange. + +Small control exchanges run inside a killable helper process, so connect, +send, read, and DNS all share one hard bound that no blocking socket state +can outlive. Large streaming Git exchanges cannot cross a process boundary; +they instead register their connection with a watchdog that force-closes the +socket when the same kind of absolute deadline expires. +""" + +from __future__ import annotations + +import base64 +import contextlib +import email.message +import http.client +import io +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass + +from gitea_api_policy import PolicyError + +MAX_CONTROL_BYTES = 128 * 1024 +MAX_PROTOCOL_BYTES = 4 * 1024 * 1024 +MAX_STREAM_SECONDS = 900.0 +READ_CHUNK = 64 * 1024 + + +class _RejectRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.URLError("redirect rejected") + + +@dataclass(frozen=True) +class Result: + """Minimal HTTP evidence returned across the process boundary.""" + + status: int + content_type: str + body: bytes + + +def _read_response(response, maximum: int, deadline: float) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(min(READ_CHUNK, maximum + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum: + raise ValueError("response too large") + return b"".join(chunks) + + +def _child_exchange(control: dict[str, object]) -> Result: + url = control.get("url") + method = control.get("method") + headers = control.get("headers") + encoded = control.get("body") + maximum = control.get("maximum") + timeout = control.get("timeout") + if ( + not isinstance(url, str) + or not isinstance(method, str) + or not isinstance(headers, dict) + or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items()) + or not isinstance(encoded, str) + or not isinstance(maximum, int) + or not 0 <= maximum <= 2 * 1024 * 1024 + or not isinstance(timeout, (int, float)) + or not 0 < timeout <= 120 + ): + raise ValueError("invalid control") + body = base64.b64decode(encoded, validate=True) if encoded else None + if body is not None and len(body) > MAX_CONTROL_BYTES: + raise ValueError("request too large") + deadline = time.monotonic() + float(timeout) + request = urllib.request.Request( + url, data=body, method=method, headers=headers # type: ignore[arg-type] + ) + opener = urllib.request.build_opener(_RejectRedirect()) + try: + response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001)) + except urllib.error.HTTPError as error: + response = error + with response: + status = getattr(response, "status", response.getcode()) + content_type = response.headers.get_content_type() + value = _read_response(response, maximum, deadline) + return Result(int(status), str(content_type), value) + + +def _child_main() -> int: + try: + raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1) + if len(raw) > MAX_CONTROL_BYTES: + raise ValueError("control too large") + control = json.loads(raw) + if not isinstance(control, dict): + raise ValueError("invalid control") + result = _child_exchange(control) + output = { + "ok": True, + "status": result.status, + "content_type": result.content_type, + "body": base64.b64encode(result.body).decode("ascii"), + } + except Exception: + output = {"ok": False} + sys.stdout.write(json.dumps(output, separators=(",", ":"))) + return 0 if output["ok"] else 1 + + +def _terminate(process: subprocess.Popen[bytes]) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + + +def exchange( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> Result: + """Execute a whole request under one monotonic deadline and size limit.""" + if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024: + raise PolicyError("HTTP control request bounds are invalid") + body = request.data or b"" + if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES: + raise PolicyError("HTTP control request exceeds the safe size limit") + control = json.dumps( + { + "url": request.full_url, + "method": request.get_method(), + "headers": dict(request.header_items()), + "body": base64.b64encode(body).decode("ascii"), + "maximum": maximum, + "timeout": timeout, + }, + separators=(",", ":"), + ).encode() + deadline = time.monotonic() + timeout + process = popen( + [sys.executable, os.path.realpath(__file__), "--child"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"}, + ) + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(process.args, timeout) + output, _error = process.communicate(control, timeout=remaining) + except subprocess.TimeoutExpired as exc: + _terminate(process) + raise PolicyError("HTTP control request deadline exceeded") from exc + if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES: + raise PolicyError("HTTP control request failed") + try: + result = json.loads(output) + if not isinstance(result, dict) or result.get("ok") is not True: + raise ValueError + status = result["status"] + content_type = result["content_type"] + encoded = result["body"] + if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str): + raise ValueError + value = base64.b64decode(encoded, validate=True) + except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc: + raise PolicyError("HTTP control helper returned invalid evidence") from exc + if len(value) > maximum: + raise PolicyError("HTTP control response exceeds the safe size limit") + return Result(status, content_type, value) + + +class _AdaptedResponse: + """Present one finished exchange through the urlopen reader interface.""" + + def __init__(self, result: Result): + self.status = result.status + self.headers = email.message.Message() + self.headers["Content-Type"] = result.content_type + self._stream = io.BytesIO(result.body) + + def read(self, limit: int = -1) -> bytes: + return self._stream.read(limit) + + def __enter__(self): + return self + + def __exit__(self, *_excinfo: object) -> bool: + return False + + +def open_bounded( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> _AdaptedResponse: + """Open one control exchange whose whole lifetime shares one deadline.""" + return _AdaptedResponse( + exchange(request, maximum=maximum, timeout=timeout, popen=popen) + ) + + +class StreamDeadline: + """Force-close tracked connections once an absolute deadline passes.""" + + def __init__(self, timeout: float, *, timer=threading.Timer): + if not 0 < timeout <= MAX_STREAM_SECONDS: + raise PolicyError("HTTP stream deadline bounds are invalid") + self._deadline = time.monotonic() + timeout + self._lock = threading.Lock() + self._connections: list[http.client.HTTPConnection] = [] + self.expired = False + self._timer = timer(timeout, self._expire) + self._timer.daemon = True + self._timer.start() + + def remaining(self) -> float: + value = self._deadline - time.monotonic() + if value <= 0 or self.expired: + raise PolicyError("HTTP stream deadline exceeded") + return value + + def _expire(self) -> None: + with self._lock: + self.expired = True + connections = list(self._connections) + for connection in connections: + # Shut the raw socket down first so a blocked send or recv wakes + # immediately; close alone leaves peers waiting on the old fd. + sock = getattr(connection, "sock", None) + if sock is not None: + with contextlib.suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + connection.close() + + def _track(self, connection: http.client.HTTPConnection): + with self._lock: + expired = self.expired + if not expired: + self._connections.append(connection) + if expired: + connection.close() + raise PolicyError("HTTP stream deadline exceeded") + return connection + + def cancel(self) -> None: + self._timer.cancel() + + def handlers(self) -> tuple[urllib.request.BaseHandler, ...]: + """Build urllib handlers whose connections obey this deadline.""" + deadline = self + + class GuardedHTTPHandler(urllib.request.HTTPHandler): + def http_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._track( + http.client.HTTPConnection(host, **kwargs) + ), + req, + ) + + class GuardedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._track( + http.client.HTTPSConnection(host, **kwargs) + ), + req, + ) + + return (GuardedHTTPHandler(), GuardedHTTPSHandler()) + + +if __name__ == "__main__": + raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2) diff --git a/services/hermes/scm-common/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py index d207cc63..3a7de241 100644 --- a/services/hermes/scm-common/scripts/gitea_api.py +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -15,6 +15,7 @@ import urllib.request from collections.abc import Callable from pathlib import Path +import deadline_http from gitea_api_policy import ( PolicyError, _draft_title, @@ -40,19 +41,16 @@ MAX_API_PATH_LENGTH = 512 RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") -class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): - """Reject every redirect before urllib can copy authentication headers.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise PolicyError("Forgejo redirects are not allowed") - - -_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler()) - - def _safe_urlopen(request: urllib.request.Request, timeout: int): - """Open one request using a handler that never follows redirects.""" - return _SAFE_OPENER.open(request, timeout=timeout) + """Open one exchange under a killable absolute wall-clock deadline. + + The helper process rejects redirects before urllib can copy + authentication headers, and its whole connect/send/read lifetime + shares one hard deadline. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py index 6c57ddfd..9a5265b6 100644 --- a/services/hermes/scm-common/scripts/scm_broker_client.py +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -7,24 +7,22 @@ import json import urllib.request from collections.abc import Callable +import deadline_http from gitea_api_policy import PolicyError BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep every broker request on its fixed in-cluster origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise PolicyError("SCM broker redirects are not allowed") - - -_OPENER = urllib.request.build_opener(RejectRedirectHandler()) - - def _open(request: urllib.request.Request, timeout: int): - return _OPENER.open(request, timeout=timeout) + """Open one broker exchange under a killable absolute deadline. + + The helper process keeps every request on its fixed in-cluster origin by + rejecting redirects before they are followed. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) def request( diff --git a/testing/tests/test_hermes_deadline_http.py b/testing/tests/test_hermes_deadline_http.py new file mode 100644 index 00000000..45fd07e4 --- /dev/null +++ b/testing/tests/test_hermes_deadline_http.py @@ -0,0 +1,309 @@ +"""Absolute-deadline contracts for the killable HTTP control helper.""" + +from __future__ import annotations + +import base64 +import io +import json +import subprocess +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Handler(BaseHTTPRequestHandler): + def _respond(self): + if self.path == "/slow": + time.sleep(10) + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:1/evil") + self.send_header("Content-Length", "0") + self.end_headers() + return + status = 404 if self.path == "/missing" else 200 + body = b"x" * 64 if self.path == "/large" else b'{"ok":true}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = _respond + do_POST = _respond + + def log_message(self, *_args): + return + + +@pytest.fixture(scope="module") +def server(): + value = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=value.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{value.server_port}" + value.shutdown() + + +def _control(url: str, **overrides): + value = { + "url": url, + "method": "GET", + "headers": {"Accept": "application/json"}, + "body": "", + "maximum": 1024, + "timeout": 10, + } + value.update(overrides) + return value + + +def test_child_exchange_round_trips_and_reports_http_errors(server): + module = _load("deadline_http") + result = module._child_exchange(_control(server + "/ok")) + assert (result.status, result.content_type) == (200, "application/json") + assert result.body == b'{"ok":true}' + + posted = module._child_exchange( + _control( + server + "/ok", + method="POST", + body=base64.b64encode(b'{"a":1}').decode(), + ) + ) + assert posted.status == 200 + + missing = module._child_exchange(_control(server + "/missing")) + assert missing.status == 404 + + +def test_child_exchange_rejects_redirects_and_oversized_responses(server): + module = _load("deadline_http") + with pytest.raises(Exception): + module._child_exchange(_control(server + "/redirect")) + with pytest.raises(ValueError, match="too large"): + module._child_exchange(_control(server + "/large", maximum=8)) + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": 1}, + {"method": None}, + {"headers": ["not", "dict"]}, + {"headers": {"Accept": 5}}, + {"body": b"raw"}, + {"maximum": "big"}, + {"maximum": -1}, + {"maximum": 4 * 1024 * 1024}, + {"timeout": "soon"}, + {"timeout": 0}, + {"timeout": 121}, + ], +) +def test_child_exchange_rejects_invalid_control_documents(overrides): + module = _load("deadline_http") + control = _control("http://127.0.0.1:1/") + control.update(overrides) + with pytest.raises(ValueError, match="invalid control"): + module._child_exchange(control) + + +def test_child_exchange_rejects_bad_or_oversized_encoded_bodies(): + module = _load("deadline_http") + with pytest.raises(ValueError): + module._child_exchange(_control("http://127.0.0.1:1/", body="!!not-b64!!")) + huge = base64.b64encode(b"x" * (module.MAX_CONTROL_BYTES + 1)).decode() + with pytest.raises(ValueError, match="request too large"): + module._child_exchange(_control("http://127.0.0.1:1/", body=huge)) + + +def test_read_response_enforces_deadline_and_size(): + module = _load("deadline_http") + + class Reader: + def __init__(self, body: bytes): + self.stream = io.BytesIO(body) + + def read(self, limit: int): + return self.stream.read(limit) + + with pytest.raises(TimeoutError): + module._read_response(Reader(b"abc"), 10, time.monotonic() - 1) + with pytest.raises(ValueError, match="too large"): + module._read_response(Reader(b"abcdef"), 4, time.monotonic() + 5) + value = module._read_response(Reader(b"abc"), 10, time.monotonic() + 5) + assert value == b"abc" + + +def _run_child_main(module, monkeypatch, raw: bytes) -> dict: + stdin = type("Stdin", (), {"buffer": io.BytesIO(raw)})() + stdout = io.StringIO() + monkeypatch.setattr(module.sys, "stdin", stdin) + monkeypatch.setattr(module.sys, "stdout", stdout) + code = module._child_main() + return code, json.loads(stdout.getvalue()) + + +def test_child_main_reports_success_and_failure_documents(monkeypatch, server): + module = _load("deadline_http") + code, output = _run_child_main( + module, monkeypatch, json.dumps(_control(server + "/ok")).encode() + ) + assert code == 0 and output["ok"] is True + assert base64.b64decode(output["body"]) == b'{"ok":true}' + + for raw in ( + b"x" * (module.MAX_CONTROL_BYTES + 1), + b"not json", + b"[1,2]", + ): + code, output = _run_child_main(module, monkeypatch, raw) + assert code == 1 and output == {"ok": False} + + +class _FakeProcess: + def __init__(self, output: bytes, returncode: int = 0): + self.args = ["fake-child"] + self.pid = 0 + self.returncode = returncode + self.output = output + self.stdin_payload = None + + def communicate(self, payload=None, timeout=None): + self.stdin_payload = payload + return self.output, b"" + + +def _fake_popen(output: bytes, returncode: int = 0): + def popen(*_args, **_kwargs): + return _FakeProcess(output, returncode) + + return popen + + +def _success_output(body: bytes, **overrides) -> bytes: + value = { + "ok": True, + "status": 200, + "content_type": "application/json", + "body": base64.b64encode(body).decode("ascii"), + } + value.update(overrides) + return json.dumps(value).encode() + + +def test_exchange_validates_bounds_and_request_body(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=1, timeout=0) + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=-1, timeout=10) + request.data = "not-bytes" + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + request.data = b"x" * (module.MAX_CONTROL_BYTES + 1) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + + +def test_exchange_returns_result_and_enforces_response_limit(): + module = _load("deadline_http") + request = urllib.request.Request( + "https://scm.bstein.dev/api/v1/x", data=b"{}", method="POST" + ) + result = module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) + assert (result.status, result.content_type, result.body) == ( + 200, + "application/json", + b"body", + ) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange( + request, + maximum=2, + timeout=10, + popen=_fake_popen(_success_output(b"body")), + ) + + +@pytest.mark.parametrize( + ("output", "returncode", "match"), + [ + (b"", 1, "request failed"), + (b"x" * (4 * 1024 * 1024 + 1), 0, "request failed"), + (b"not json", 0, "invalid evidence"), + (b'{"ok":false}', 0, "invalid evidence"), + (b"[1]", 0, "invalid evidence"), + (b'{"ok":true}', 0, "invalid evidence"), + (b'{"ok":true,"status":"200","content_type":"a","body":""}', 0, "invalid evidence"), + (b'{"ok":true,"status":200,"content_type":"a","body":"!!"}', 0, "invalid evidence"), + ], +) +def test_exchange_rejects_invalid_helper_evidence(output, returncode, match): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match=match): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(output, returncode) + ) + + +def test_exchange_kills_the_helper_when_the_deadline_expires(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/slow") + started = time.monotonic() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange(request, maximum=64, timeout=1.0) + assert time.monotonic() - started < 5 + + +def test_exchange_fails_before_spawn_wait_when_time_is_exhausted(monkeypatch): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + killed = [] + monkeypatch.setattr(module.os, "killpg", lambda pid, sig: killed.append((pid, sig))) + clock = iter([0.0, 1_000_000.0, 1_000_000.0, 1_000_000.0]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(clock)) + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"")) + ) + assert killed + + +def test_terminate_tolerates_already_finished_helpers(): + module = _load("deadline_http") + process = subprocess.Popen( + ["/bin/true"], start_new_session=True, stdout=subprocess.DEVNULL + ) + process.wait() + module._terminate(process) + assert process.returncode == 0 + + +def test_child_entrypoint_round_trip_through_real_subprocess(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/ok") + result = module.exchange(request, maximum=1024, timeout=20) + assert result.status == 200 and result.body == b'{"ok":true}' + + +def test_open_bounded_adapts_result_to_the_urlopen_interface(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with module.open_bounded( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) as response: + assert response.status == 200 + assert response.headers.get_content_type() == "application/json" + assert response.read(2) == b"bo" + assert response.read() == b"dy" diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py index f811827c..da720878 100644 --- a/testing/tests/test_hermes_gitea_internal_coverage.py +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -37,16 +37,21 @@ def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): module.configured_base_url() -def test_safe_urlopen_delegates_only_to_redirect_rejecting_opener(monkeypatch): +def test_safe_urlopen_delegates_only_to_deadline_bounded_exchange(monkeypatch): module = _load() sentinel = object() monkeypatch.setattr( - module._SAFE_OPENER, - "open", - lambda request, timeout: (request, timeout, sentinel), + module.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), ) request = object() - assert module._safe_urlopen(request, 7) == (request, 7, sentinel) + assert module._safe_urlopen(request, 7) == ( + request, + module.MAX_RESPONSE_BYTES, + 7, + sentinel, + ) def test_api_target_and_request_body_edge_paths(monkeypatch): diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index ae419e10..4ddb9830 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -2,21 +2,22 @@ from __future__ import annotations +import urllib.error import urllib.request import pytest from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load -def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): +def test_default_opener_rejects_redirects_under_an_absolute_deadline(monkeypatch): client = _load() source = urllib.request.Request( "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", headers={"Authorization": "token redirect-sentinel"}, ) - with pytest.raises(client.PolicyError, match="redirects are not allowed") as exc: - client.RejectRedirectHandler().redirect_request( + with pytest.raises(urllib.error.URLError) as exc: + client.deadline_http._RejectRedirect().redirect_request( source, None, 302, @@ -26,10 +27,16 @@ def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): ) assert "redirect-sentinel" not in str(exc.value) - assert any( - isinstance(handler, client.RejectRedirectHandler) - for handler in client._SAFE_OPENER.handlers + seen = [] + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: seen.append( + (request, maximum, timeout) + ), ) + client._safe_urlopen(source, 30) + assert seen == [(source, client.MAX_RESPONSE_BYTES, 30)] @pytest.mark.parametrize( diff --git a/testing/tests/test_hermes_scm_broker_client_coverage.py b/testing/tests/test_hermes_scm_broker_client_coverage.py index ca6a357d..9fa65fa6 100644 --- a/testing/tests/test_hermes_scm_broker_client_coverage.py +++ b/testing/tests/test_hermes_scm_broker_client_coverage.py @@ -9,17 +9,20 @@ import pytest from testing.tests.test_hermes_scm_broker_support import Response, _load -def test_client_redirect_and_default_opener_are_fail_closed(monkeypatch): +def test_client_default_opener_uses_deadline_bounded_exchange(monkeypatch): client = _load("scm_broker_client") - with pytest.raises(client.PolicyError, match="redirects"): - client.RejectRedirectHandler().redirect_request( - object(), None, 302, "redirect", {}, "http://elsewhere.invalid" - ) sentinel = object() monkeypatch.setattr( - client._OPENER, "open", lambda request, timeout: (request, timeout, sentinel) + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + assert client._open("request", 4) == ( + "request", + client.MAX_RESPONSE_BYTES, + 4, + sentinel, ) - assert client._open("request", 4) == ("request", 4, sentinel) @pytest.mark.parametrize( From 340ec58b68b3e8e1f46a1337b4855acedc1bc6ac Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 31/46] hermes: quarantine-scan pushed git objects Inflate every receive-pack object under strict pack, size, and checksum bounds, resolve deltas against in-pack bases only, and scan the real decompressed payloads for runtime-token forms, private keys, SSH key material, and known provider token formats. Thin packs are rejected so no pushed content escapes the scan, and upstream Git exchanges now run under one absolute stream deadline. Co-Authored-By: Claude Fable 5 --- services/hermes/agent-configmap.yaml | 4 +- services/hermes/scm-common/kustomization.yaml | 3 + .../scm-common/scripts/git_pack_objects.py | 308 ++++++++++++++++++ .../scm-common/scripts/receive_pack_scan.py | 143 ++++++++ .../hermes/scm-common/scripts/scm_broker.py | 103 ++---- testing/tests/test_hermes_deadline_stream.py | 217 ++++++++++++ testing/tests/test_hermes_git_pack_objects.py | 288 ++++++++++++++++ .../tests/test_hermes_receive_pack_scan.py | 217 ++++++++++++ ...est_hermes_scm_broker_internal_coverage.py | 29 +- .../tests/test_hermes_scm_broker_streaming.py | 2 +- .../tests/test_hermes_scm_broker_support.py | 33 +- 11 files changed, 1261 insertions(+), 86 deletions(-) create mode 100644 services/hermes/scm-common/scripts/git_pack_objects.py create mode 100644 services/hermes/scm-common/scripts/receive_pack_scan.py create mode 100644 testing/tests/test_hermes_deadline_stream.py create mode 100644 testing/tests/test_hermes_git_pack_objects.py create mode 100644 testing/tests/test_hermes_receive_pack_scan.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index b53b2c92..28d520fb 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -320,7 +320,9 @@ data: unauthenticated Gitea HTTP 404 as a missing private repository. Use brokered Git for clone, fetch, and creation of a new namespaced feature branch. Existing-ref updates, protected refs, deletion, and force-push are - rejected. For bounded + rejected. The broker inflates and scans every pushed object, so thin + packs are rejected; always push with `git push --no-thin` so the pack + is self-contained. For bounded repository/pull-request evidence or to create a review-ready draft PR, load `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The client carries no repository credential; a separate least-authority diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml index 17dc1da6..a4756ccd 100644 --- a/services/hermes/scm-common/kustomization.yaml +++ b/services/hermes/scm-common/kustomization.yaml @@ -4,8 +4,11 @@ kind: Kustomization configMapGenerator: - name: hermes-scm-boundary files: + - deadline_http.py=scripts/deadline_http.py + - git_pack_objects.py=scripts/git_pack_objects.py - gitea_api.py=scripts/gitea_api.py - gitea_api_policy.py=scripts/gitea_api_policy.py + - receive_pack_scan.py=scripts/receive_pack_scan.py - scm_broker.py=scripts/scm_broker.py - scm_broker_io.py=scripts/scm_broker_io.py - scm_broker_server.py=scripts/scm_broker_server.py diff --git a/services/hermes/scm-common/scripts/git_pack_objects.py b/services/hermes/scm-common/scripts/git_pack_objects.py new file mode 100644 index 00000000..d9a3e715 --- /dev/null +++ b/services/hermes/scm-common/scripts/git_pack_objects.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Bounded Git pack inflation and delta resolution for quarantine scanning. + +Pushed objects arrive zlib-compressed inside a packfile, so any content +screening must inflate the real payloads first. This module unpacks one +self-contained pack under explicit object, size, and checksum bounds and +resolves every delta against bases inside the same pack. Thin packs are +rejected outright: a delta against a base the broker cannot see would leave +part of the pushed content unscanned. +""" + +from __future__ import annotations + +import hashlib +import tempfile +import zlib +from typing import BinaryIO + +from gitea_api_policy import PolicyError + +OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"} +OFS_DELTA = 6 +REF_DELTA = 7 +MAX_PACK_OBJECTS = 50_000 +MAX_OBJECT_BYTES = 32 * 1024 * 1024 +MAX_TOTAL_BYTES = 256 * 1024 * 1024 +MAX_RESOLVE_PASSES = 64 +SPOOL_MEMORY_LIMIT = 256 * 1024 +READ_CHUNK = 64 * 1024 + + +def _read_exact(stream: BinaryIO, size: int) -> bytes: + value = stream.read(size) + if len(value) != size: + raise PolicyError("Git pack data ended early") + return value + + +def _object_header(stream: BinaryIO) -> tuple[int, int]: + """Parse one object's type and declared inflated size.""" + byte = _read_exact(stream, 1)[0] + type_code = (byte >> 4) & 0x7 + size = byte & 0x0F + shift = 4 + while byte & 0x80: + if shift > 60: + raise PolicyError("Git object header is invalid") + byte = _read_exact(stream, 1)[0] + size |= (byte & 0x7F) << shift + shift += 7 + return type_code, size + + +def _delta_distance(stream: BinaryIO) -> int: + """Parse one ofs-delta base distance.""" + byte = _read_exact(stream, 1)[0] + value = byte & 0x7F + while byte & 0x80: + byte = _read_exact(stream, 1)[0] + value = ((value + 1) << 7) | (byte & 0x7F) + if value > 1 << 48: + raise PolicyError("Git delta base distance is invalid") + return value + + +def _inflate(stream: BinaryIO, declared: int) -> BinaryIO: + """Inflate one object into a quarantine spool, bounded by its header.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + decompressor = zlib.decompressobj() + produced = 0 + try: + while not decompressor.eof: + data = stream.read(READ_CHUNK) + if not data: + raise PolicyError("Git pack object data ended early") + while True: + value = decompressor.decompress(data, READ_CHUNK) + produced += len(value) + if produced > declared: + raise PolicyError("Git object size does not match its header") + spool.write(value) + data = decompressor.unconsumed_tail + if decompressor.eof or not data: + break + if decompressor.unused_data: + stream.seek(-len(decompressor.unused_data), 1) + if produced != declared: + raise PolicyError("Git object size does not match its header") + spool.seek(0) + return spool + except zlib.error as exc: + spool.close() + raise PolicyError("Git pack object data is corrupt") from exc + except Exception: + spool.close() + raise + + +def _delta_size(delta: bytes, position: int) -> tuple[int, int]: + size = 0 + shift = 0 + while True: + if position >= len(delta) or shift > 60: + raise PolicyError("Git delta header is invalid") + byte = delta[position] + position += 1 + size |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return size, position + + +def _apply_delta(base: bytes, delta: bytes) -> bytes: + """Apply one Git delta and require exact declared sizes throughout.""" + source_size, position = _delta_size(delta, 0) + target_size, position = _delta_size(delta, position) + if source_size != len(base) or target_size > MAX_OBJECT_BYTES: + raise PolicyError("Git delta sizes are invalid") + parts: list[bytes] = [] + produced = 0 + while position < len(delta): + opcode = delta[position] + position += 1 + if opcode & 0x80: + operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit] + if position + len(operands) > len(delta): + raise PolicyError("Git delta copy is truncated") + offset = 0 + size = 0 + for shift, bit in enumerate((1, 2, 4, 8)): + if opcode & bit: + offset |= delta[position] << (8 * shift) + position += 1 + for shift, bit in enumerate((16, 32, 64)): + if opcode & bit: + size |= delta[position] << (8 * shift) + position += 1 + size = size or 0x10000 + if offset + size > len(base): + raise PolicyError("Git delta copy is out of bounds") + parts.append(base[offset : offset + size]) + produced += size + elif opcode: + if position + opcode > len(delta): + raise PolicyError("Git delta insert is truncated") + parts.append(delta[position : position + opcode]) + position += opcode + produced += opcode + else: + raise PolicyError("Git delta instruction is invalid") + if produced > target_size: + raise PolicyError("Git delta result exceeds its declared size") + if produced != target_size: + raise PolicyError("Git delta result size is invalid") + return b"".join(parts) + + +def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes: + digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size) + payload.seek(0) + while True: + chunk = payload.read(READ_CHUNK) + if not chunk: + break + digest.update(chunk) + payload.seek(0) + return digest.digest() + + +def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None: + trailer = _read_exact(stream, 20) + stream.seek(start) + digest = hashlib.sha1() + remaining = end - start + while remaining: + chunk = stream.read(min(READ_CHUNK, remaining)) + digest.update(chunk) + remaining -= len(chunk) + if digest.digest() != trailer: + raise PolicyError("Git pack checksum is invalid") + stream.seek(end + 20) + + +class _Entry: + """One pack entry: either resolved content or an unresolved delta.""" + + def __init__(self, type_code: int, payload: BinaryIO, size: int, base): + self.type_code = type_code + self.payload = payload + self.size = size + self.base = base + + +def _resolve(entries: dict[int, _Entry], total: list[int]) -> None: + """Resolve every delta against in-pack bases only, to a fixed point.""" + by_sha: dict[bytes, int] = {} + for offset, entry in entries.items(): + if entry.base is None: + by_sha.setdefault( + _payload_sha(entry.type_code, entry.payload, entry.size), offset + ) + for _ in range(MAX_RESOLVE_PASSES): + pending = [item for item in entries.items() if item[1].base is not None] + if not pending: + return + progressed = False + for offset, entry in pending: + base_key = entry.base + base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key) + base = entries.get(base_offset) if base_offset is not None else None + if base is None or base.base is not None: + if isinstance(base_key, int) and base is None: + raise PolicyError("Git delta base offset is invalid") + continue + base.payload.seek(0) + entry.payload.seek(0) + value = _apply_delta(base.payload.read(), entry.payload.read()) + total[0] += len(value) + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + entry.payload.close() + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + spool.write(value) + spool.seek(0) + entry.type_code = base.type_code + entry.payload = spool + entry.size = len(value) + entry.base = None + by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset) + progressed = True + if not progressed: + raise PolicyError("Git delta base is outside the pack; push full packs") + raise PolicyError("Git delta chains exceed the safe resolution limit") + + +def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]: + """Inflate one self-contained pack and return every resolved payload.""" + start = stream.tell() + header = _read_exact(stream, 12) + version = int.from_bytes(header[4:8], "big") + count = int.from_bytes(header[8:12], "big") + if header[:4] != b"PACK" or version not in {2, 3}: + raise PolicyError("Git pack header is invalid") + if count > MAX_PACK_OBJECTS: + raise PolicyError("Git pack contains too many objects") + entries: dict[int, _Entry] = {} + total = [0] + try: + for _ in range(count): + offset = stream.tell() - start + type_code, declared = _object_header(stream) + if declared > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + total[0] += declared + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + base: int | bytes | None = None + if type_code == OFS_DELTA: + distance = _delta_distance(stream) + base = offset - distance + if base < 0: + raise PolicyError("Git delta base offset is invalid") + elif type_code == REF_DELTA: + base = _read_exact(stream, 20) + elif type_code not in OBJECT_TYPE_NAMES: + raise PolicyError("Git pack object type is invalid") + entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base) + _verify_checksum(stream, start, stream.tell()) + _resolve(entries, total) + except Exception: + for entry in entries.values(): + entry.payload.close() + raise + for entry in entries.values(): + entry.payload.seek(0) + return [ + (entries[offset].type_code, entries[offset].payload) + for offset in sorted(entries) + ] + + +def parse_loose_object(value: bytes) -> tuple[int, bytes]: + """Inflate one quarantined loose object and return its typed payload.""" + if len(value) > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + decompressor = zlib.decompressobj() + try: + inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1) + except zlib.error as exc: + raise PolicyError("Git loose object data is corrupt") from exc + if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES: + raise PolicyError("Git loose object framing is invalid") + kind, separator, remainder = inflated.partition(b"\x00") + name, space, size = kind.partition(b" ") + types = {label: code for code, label in OBJECT_TYPE_NAMES.items()} + if ( + not separator + or not space + or name not in types + or not size.isdigit() + or int(size) != len(remainder) + ): + raise PolicyError("Git loose object header is invalid") + return types[name], remainder diff --git a/services/hermes/scm-common/scripts/receive_pack_scan.py b/services/hermes/scm-common/scripts/receive_pack_scan.py new file mode 100644 index 00000000..76980880 --- /dev/null +++ b/services/hermes/scm-common/scripts/receive_pack_scan.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Validate Git receive-pack requests and scan pushed object content. + +Ref commands may only create new namespaced feature branches. Every pushed +object is then inflated in quarantine and its decompressed payload scanned, +so credentials cannot ride through the broker inside compressed pack data. +The screening is a fail-closed accident barrier for credential-shaped +content: exact runtime-token forms, private keys, SSH key material, and +known provider token formats. Entropy heuristics are deliberately excluded +so SOPS ciphertext, pinned digests, and lock files remain pushable; a short +secret disguised as prose is not reliably detectable and must never be +supplied by callers. +""" + +from __future__ import annotations + +import io +import re +from typing import BinaryIO + +import git_pack_objects +from gitea_api_policy import ( + STANDALONE_SECRET_PATTERNS, + PolicyError, + _validate_ref, +) + +ZERO_SHA = b"0" * 40 +MAX_PUSH_COMMANDS = 16 +SCAN_CHUNK = 256 * 1024 +SCAN_CARRY = 8 * 1024 +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) +PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z") +CONTENT_SECRET_PATTERNS = tuple( + re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE) + for pattern in STANDALONE_SECRET_PATTERNS +) + ( + re.compile(rb"(?i)(? bytes | None: + """Read one pkt-line; None marks the flush packet before the pack.""" + header = stream.read(4) + if not header: + raise PolicyError("Git receive-pack request omitted its command terminator") + if not PKT_HEADER_RE.fullmatch(header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + return None + if size < 4: + raise PolicyError("Git receive-pack command framing is invalid") + value = stream.read(size - 4) + if len(value) != size - 4: + raise PolicyError("Git receive-pack command framing is invalid") + return value + + +def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None: + """Permit only creation of one new, namespaced feature branch.""" + if any(form in line for form in forbidden): + raise PolicyError("Git request contains runtime credential material") + command = line.rstrip(b"\n").split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + + +def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + """Scan one decompressed object payload across chunk boundaries.""" + carry = b"" + while True: + chunk = payload.read(SCAN_CHUNK) + if not chunk: + return + window = carry + chunk + if any(form in window for form in forbidden): + raise PolicyError("Git push contains runtime credential material") + if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS): + raise PolicyError("Git push contains credential-shaped content") + carry = window[-SCAN_CARRY:] + + +def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + objects = git_pack_objects.unpack_objects(stream) + try: + if stream.read(1): + raise PolicyError("Git request has bytes after its pack") + for _type_code, payload in objects: + _scan_payload(payload, forbidden) + finally: + for _type_code, payload in objects: + payload.close() + + +def validate_receive_pack( + body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...] +) -> None: + """Validate one whole receive-pack request: commands, pack, and content.""" + stream = io.BytesIO(body) if isinstance(body, bytes) else body + stream.seek(0) + try: + commands = 0 + while True: + line = _read_pkt_line(stream) + if line is None: + break + _validate_command(line, token, forbidden) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + _scan_pack(stream, forbidden) + finally: + stream.seek(0) + + +def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int: + """Scan one quarantined loose object and return its parsed type code.""" + type_code, payload = git_pack_objects.parse_loose_object(value) + _scan_payload(io.BytesIO(payload), forbidden) + return type_code diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py index a0c4043c..a9cb3685 100644 --- a/services/hermes/scm-common/scripts/scm_broker.py +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -16,6 +16,7 @@ import urllib.request from http.server import BaseHTTPRequestHandler from typing import BinaryIO +import deadline_http from gitea_api import ( CANONICAL_BASE_URL, PolicyError, @@ -23,7 +24,8 @@ from gitea_api import ( read, read_token, ) -from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo +from gitea_api_policy import _reject_forbidden, _validate_repo +from receive_pack_scan import validate_receive_pack from scm_broker_io import RejectRedirect, response_status as _status, spool_response from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer @@ -32,24 +34,17 @@ GIT_USER = "hermes-automation" MAX_CONTROL_BODY = 64 * 1024 MAX_GIT_REQUEST = 128 * 1024 * 1024 MAX_GIT_RESPONSE = 128 * 1024 * 1024 -MAX_PUSH_COMMANDS = 16 MAX_HEADERS = 32 MAX_HEADER_BYTES = 16 * 1024 SPOOL_MEMORY_LIMIT = 1024 * 1024 STREAM_CHUNK = 64 * 1024 INBOUND_HEADER_TIMEOUT = 10.0 INBOUND_BODY_TIMEOUT = 120.0 -ZERO_SHA = b"0" * 40 +UPSTREAM_DEADLINE_SECONDS = 300.0 GIT_PATH_RE = re.compile( r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" ) -FEATURE_REF_RE = re.compile( - r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" -) - - -UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect()) def _read_bounded( @@ -126,14 +121,16 @@ def _spool_bounded( raise -def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]: +def _spool_response( + response, maximum: int, token: str, deadline_seconds: float +) -> tuple[BinaryIO, int]: return spool_response( response, maximum, _credential_forms(token), memory_limit=SPOOL_MEMORY_LIMIT, chunk_size=STREAM_CHUNK, - deadline_seconds=INBOUND_BODY_TIMEOUT, + deadline_seconds=deadline_seconds, ) @@ -211,61 +208,14 @@ def _credential_forms(token: str) -> tuple[bytes, ...]: return token.encode("utf-8"), basic, b"Basic " + basic -def _reject_credential_bytes(value: bytes, token: str, context: str) -> None: - if any(form in value for form in _credential_forms(token)): - raise PolicyError(f"{context} contains runtime credential material") - - -def _receive_prefix(body: bytes | BinaryIO) -> bytes: - if isinstance(body, bytes): - return body - position = body.tell() - try: - body.seek(0) - return body.read(64 * 1024) - finally: - body.seek(position) - - def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: - """Permit only creation of new, namespaced feature branches.""" - body = _receive_prefix(body) - _reject_credential_bytes(body, token, "Git request") - position = 0 - commands = 0 - while position + 4 <= len(body): - header = body[position : position + 4] - if not re.fullmatch(rb"[0-9a-f]{4}", header): - raise PolicyError("Git receive-pack command framing is invalid") - size = int(header, 16) - if size == 0: - if commands == 0: - raise PolicyError("Git receive-pack request has no ref command") - return - if size < 4 or position + size > len(body): - raise PolicyError("Git receive-pack command framing is invalid") - command = body[position + 4 : position + size].rstrip(b"\n") - command = command.split(b"\x00", 1)[0] - fields = command.split(b" ") - if len(fields) != 3 or not all( - re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] - ): - raise PolicyError("Git receive-pack ref command is invalid") - old_sha, new_sha, raw_ref = fields - if old_sha != ZERO_SHA or new_sha == ZERO_SHA: - raise PolicyError("Git broker permits only new feature-branch creation") - try: - ref = raw_ref.decode("ascii") - except UnicodeDecodeError as exc: - raise PolicyError("Git ref must be canonical ASCII") from exc - if not FEATURE_REF_RE.fullmatch(ref): - raise PolicyError("Git push is limited to namespaced feature branches") - _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) - commands += 1 - if commands > MAX_PUSH_COMMANDS: - raise PolicyError("Git push contains too many ref commands") - position += size - raise PolicyError("Git receive-pack request omitted its command terminator") + """Permit only vetted new-feature-branch pushes with scanned contents.""" + validate_receive_pack(body, token, _credential_forms(token)) + + +def _guarded_opener(guard: deadline_http.StreamDeadline): + """Build an opener whose connection obeys one absolute stream deadline.""" + return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open def _upstream_git_request( @@ -278,7 +228,7 @@ def _upstream_git_request( token: str, body_length: int | None = None, stream_result: bool = False, - opener=UPSTREAM_OPENER.open, + opener=None, ) -> bytes | tuple[BinaryIO, int]: credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} @@ -296,12 +246,21 @@ def _upstream_git_request( method=method, headers=headers, ) - with opener(request, timeout=120) as response: - if _status(response) != 200: - raise PolicyError("Git upstream returned an unexpected HTTP status") - if response.headers.get_content_type() != expected_type: - raise PolicyError("Git upstream returned an unexpected response type") - result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token) + # One wall-clock budget covers connect, send, headers, and body read; the + # watchdog closes the upstream socket if any phase overstays it. + guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS) + try: + open_upstream = opener if opener is not None else _guarded_opener(guard) + with open_upstream(request, timeout=guard.remaining()) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response( + response, MAX_GIT_RESPONSE, token, guard.remaining() + ) + finally: + guard.cancel() if stream_result: return result, result_length try: diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py new file mode 100644 index 00000000..d4f97efe --- /dev/null +++ b/testing/tests/test_hermes_deadline_stream.py @@ -0,0 +1,217 @@ +"""Watchdog contracts for absolute deadlines on streaming Git exchanges.""" + +from __future__ import annotations + +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_stream_deadline_validates_bounds_and_counts_down(): + module = _load("deadline_http") + for timeout in (0, -1, module.MAX_STREAM_SECONDS + 1): + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.StreamDeadline(timeout) + guard = module.StreamDeadline(30) + try: + first = guard.remaining() + assert 0 < first <= 30 + finally: + guard.cancel() + + +class _FakeTimer: + def __init__(self, timeout, callback): + self.timeout = timeout + self.callback = callback + self.daemon = False + self.started = False + self.cancelled = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + +class _FakeSocket: + def __init__(self, fail: bool = False): + self.fail = fail + self.shut = False + + def shutdown(self, _how): + if self.fail: + raise OSError("already gone") + self.shut = True + + +class _FakeConnection: + def __init__(self, sock=None): + self.sock = sock + self.closed = False + + def close(self): + self.closed = True + + +def test_expiry_shuts_down_and_closes_every_tracked_connection(): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + assert guard._timer.started and guard._timer.daemon + healthy = _FakeConnection(_FakeSocket()) + broken = _FakeConnection(_FakeSocket(fail=True)) + bare = _FakeConnection() + for connection in (healthy, broken, bare): + assert guard._track(connection) is connection + guard._expire() + assert guard.expired + assert healthy.sock.shut and healthy.closed + assert broken.closed and not broken.sock.shut + assert bare.closed + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard.remaining() + late = _FakeConnection() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard._track(late) + assert late.closed + guard.cancel() + assert guard._timer.cancelled + + +def test_handlers_build_guarded_connections_without_network(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + http_handler, https_handler = guard.handlers() + assert isinstance(http_handler, urllib.request.HTTPHandler) + assert isinstance(https_handler, urllib.request.HTTPSHandler) + for handler, opener_name in ( + (http_handler, "http_open"), + (https_handler, "https_open"), + ): + seen = {} + monkeypatch.setattr( + handler, + "do_open", + lambda factory, req, seen=seen: seen.update(factory=factory, req=req), + ) + getattr(handler, opener_name)("request") + connection = seen["factory"]("127.0.0.1", timeout=1) + assert connection in guard._connections + connection.close() + guard.cancel() + + +class _TrickleHandler(BaseHTTPRequestHandler): + def do_GET(self): + time.sleep(10) + + def log_message(self, *_args): + return + + +def test_watchdog_interrupts_a_hung_streaming_exchange(): + module = _load("deadline_http") + server = HTTPServer(("127.0.0.1", 0), _TrickleHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + guard = module.StreamDeadline(1.0) + opener = urllib.request.build_opener(*guard.handlers()) + started = time.monotonic() + try: + with pytest.raises(OSError): + opener.open( + f"http://127.0.0.1:{server.server_port}/slow", timeout=30 + ).read() + assert time.monotonic() - started < 5 + assert guard.expired + finally: + guard.cancel() + server.shutdown() + + +def test_upstream_git_request_runs_under_one_wall_clock_budget(monkeypatch): + broker = _load("scm_broker") + seen = {} + + class RecordingGuard: + def __init__(self, timeout, **_kwargs): + seen["timeout"] = timeout + self.cancelled = False + seen["guard"] = self + + def remaining(self): + return 12.5 + + def cancel(self): + self.cancelled = True + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", RecordingGuard) + + def opener(request, timeout): + seen["opener_timeout"] = timeout + return Response(b"result", content_type="application/x-git-upload-pack-result") + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + assert result == b"result" + assert seen["timeout"] == broker.UPSTREAM_DEADLINE_SECONDS + assert seen["opener_timeout"] == 12.5 + assert seen["guard"].cancelled + + +def test_upstream_git_request_cancels_guard_when_deadline_already_passed(monkeypatch): + broker = _load("scm_broker") + cancelled = [] + + class ExpiredGuard: + def __init__(self, _timeout, **_kwargs): + pass + + def remaining(self): + raise broker.PolicyError("HTTP stream deadline exceeded") + + def cancel(self): + cancelled.append(True) + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", ExpiredGuard) + with pytest.raises(broker.PolicyError, match="deadline exceeded"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=lambda *_a, **_k: None, + ) + assert cancelled == [True] + + +def test_guarded_opener_wires_redirect_rejection_and_deadline_handlers(): + broker = _load("scm_broker") + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + opener = broker._guarded_opener(guard) + assert callable(opener) + handlers = opener.__self__.handlers + assert any(isinstance(item, broker.RejectRedirect) for item in handlers) + assert any(isinstance(item, urllib.request.HTTPSHandler) for item in handlers) + guard.cancel() diff --git a/testing/tests/test_hermes_git_pack_objects.py b/testing/tests/test_hermes_git_pack_objects.py new file mode 100644 index 00000000..f3a249ec --- /dev/null +++ b/testing/tests/test_hermes_git_pack_objects.py @@ -0,0 +1,288 @@ +"""Bounded pack inflation and delta-resolution contracts for pushed objects.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _load, + _object_entry, + _pack_of, +) + + +def _delta_header(type_code: int, size: int, suffix: bytes = b"") -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + suffix + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _ofs_distance(value: int) -> bytes: + out = [value & 0x7F] + value >>= 7 + while value: + value -= 1 + out.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(out) + + +def _delta(source: bytes, target_size: int, instructions: bytes) -> bytes: + return _size_varint(len(source)) + _size_varint(target_size) + instructions + + +def _blob_sha(payload: bytes) -> bytes: + return hashlib.sha1(b"blob %d\x00" % len(payload) + payload).digest() + + +def _read_all(objects): + try: + return [(code, payload.read()) for code, payload in objects] + finally: + for _code, payload in objects: + payload.close() + + +def test_unpack_returns_typed_payloads_for_every_base_object_kind(): + module = _load("git_pack_objects") + entries = [ + _object_entry(1, b"tree 1\n"), + _object_entry(2, b"\x00tree-bytes"), + _object_entry(3, b"blob contents"), + _object_entry(4, b"tag contents"), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert result == [ + (1, b"tree 1\n"), + (2, b"\x00tree-bytes"), + (3, b"blob contents"), + (4, b"tag contents"), + ] + + +def test_unpack_accepts_the_empty_pack(): + module = _load("git_pack_objects") + assert module.unpack_objects(io.BytesIO(_pack_of([]))) == [] + + +def test_ofs_delta_resolves_against_its_in_pack_base(): + module = _load("git_pack_objects") + base = b"base payload for copying" + base_entry = _object_entry(3, base) + # Copy the first 4 bytes, then insert new bytes. + instructions = bytes([0x80 | 0x10, 4]) + bytes([5]) + b"+tail" + delta = _delta(base, 9, instructions) + delta_entry = _delta_header( + 6, len(delta), _ofs_distance(len(base_entry)) + ) + zlib.compress(delta) + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of([base_entry, delta_entry])))) + assert result == [(3, base), (3, b"base+tail")] + + +def test_ref_delta_resolves_by_sha_and_supports_chains(): + module = _load("git_pack_objects") + base = b"chain base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert [payload for _code, payload in result] == [base, b"one!", b"two!"] + + +def test_thin_pack_ref_delta_with_external_base_is_rejected(): + module = _load("git_pack_objects") + delta = _delta(b"absent", 1, bytes([1]) + b"x") + entry = _delta_header(7, len(delta), b"\xaa" * 20) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="push full packs"): + module.unpack_objects(io.BytesIO(_pack_of([entry]))) + + +def test_ofs_delta_bad_offsets_fail_closed(): + module = _load("git_pack_objects") + base_entry = _object_entry(3, b"base") + delta = _delta(b"base", 1, bytes([1]) + b"x") + # Distance beyond the pack start. + early = _delta_header(6, len(delta), _ofs_distance(999)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, early]))) + # Distance landing between object boundaries. + misaligned = _delta_header(6, len(delta), _ofs_distance(3)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, misaligned]))) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda pack: b"JUNK" + pack[4:], "header is invalid"), + (lambda pack: pack[:4] + (9).to_bytes(4, "big") + pack[8:], "header is invalid"), + (lambda pack: pack[:8] + (2).to_bytes(4, "big") + pack[12:], "Git"), + (lambda pack: pack[:-1], "ended early"), + (lambda pack: pack[:-20] + b"\x00" * 20, "checksum is invalid"), + (lambda pack: pack[:11], "ended early"), + ], +) +def test_malformed_pack_containers_fail_closed(mutate, match): + module = _load("git_pack_objects") + pack = _pack_of([_object_entry(3, b"payload")]) + with pytest.raises(module.PolicyError, match=match): + module.unpack_objects(io.BytesIO(mutate(pack))) + + +def test_object_count_and_type_bounds_fail_closed(monkeypatch): + module = _load("git_pack_objects") + body = b"PACK" + (2).to_bytes(4, "big") + (module.MAX_PACK_OBJECTS + 1).to_bytes( + 4, "big" + ) + with pytest.raises(module.PolicyError, match="too many objects"): + module.unpack_objects(io.BytesIO(body + hashlib.sha1(body).digest())) + reserved = _delta_header(5, 1) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="object type is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([reserved]))) + + +def test_declared_size_mismatches_and_zlib_bombs_fail_closed(monkeypatch): + module = _load("git_pack_objects") + longer = _delta_header(3, 2) + zlib.compress(b"longer-than-two") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([longer]))) + shorter = _delta_header(3, 9) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([shorter]))) + corrupt = _delta_header(3, 4) + b"\x00not-zlib\x00" + with pytest.raises(module.PolicyError, match="corrupt"): + module.unpack_objects(io.BytesIO(_pack_of([corrupt]))) + + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.unpack_objects(io.BytesIO(_pack_of([_object_entry(3, b"12345")]))) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 1024) + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", 8) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects( + io.BytesIO(_pack_of([_object_entry(3, b"12345"), _object_entry(3, b"12345")])) + ) + + +def test_object_and_distance_varints_are_bounded(): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match="object header is invalid"): + module._object_header(io.BytesIO(b"\xff" * 12)) + with pytest.raises(module.PolicyError, match="ended early"): + module._object_header(io.BytesIO(b"\x80")) + with pytest.raises(module.PolicyError, match="distance is invalid"): + module._delta_distance(io.BytesIO(b"\xff" * 12)) + + +@pytest.mark.parametrize( + ("base", "delta", "match"), + [ + (b"base", b"", "delta header is invalid"), + (b"base", _size_varint(4), "delta header is invalid"), + (b"base", _size_varint(9) + _size_varint(1) + bytes([1]) + b"x", "sizes are invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0]), "instruction is invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([5]) + b"ab", "insert is truncated"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x10]), "copy is truncated"), + ( + b"base", + _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x01 | 0x10, 3, 9]), + "copy is out of bounds", + ), + (b"base", _size_varint(4) + _size_varint(1) + bytes([2]) + b"ab", "exceeds its declared size"), + (b"base", _size_varint(4) + _size_varint(3) + bytes([2]) + b"ab", "result size is invalid"), + ], +) +def test_delta_application_rejects_malformed_instructions(base, delta, match): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match=match): + module._apply_delta(base, delta) + + +def test_delta_copy_with_zero_size_field_means_the_git_default(): + module = _load("git_pack_objects") + base = b"z" * 0x10000 + delta = _delta(base, 0x10000, bytes([0x80 | 0x10 | 0x20, 0, 0])) + assert module._apply_delta(base, delta) == base + + +def test_delta_target_size_cap_applies(monkeypatch): + module = _load("git_pack_objects") + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + delta = _delta(b"base", 5, bytes([5]) + b"abcde") + with pytest.raises(module.PolicyError, match="sizes are invalid"): + module._apply_delta(b"base", delta) + + +def test_resolution_pass_and_total_caps_apply(monkeypatch): + module = _load("git_pack_objects") + base = b"pass base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + ] + pack = _pack_of(entries) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 1) + with pytest.raises(module.PolicyError, match="resolution limit"): + module.unpack_objects(io.BytesIO(pack)) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 64) + result = _read_all(module.unpack_objects(io.BytesIO(pack))) + assert [payload for _code, payload in result] == [b"two!", base, b"one!"] + + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", len(base) + len(first) + len(second) + 1) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects(io.BytesIO(pack)) + + +def test_parse_loose_object_round_trip_and_bounds(monkeypatch): + module = _load("git_pack_objects") + payload = b"loose blob" + value = zlib.compress(b"blob %d\x00" % len(payload) + payload) + assert module.parse_loose_object(value) == (3, payload) + + with pytest.raises(module.PolicyError, match="corrupt"): + module.parse_loose_object(b"\x00junk") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value + b"trailing") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value[:-4]) + for header in ( + b"noodle 3\x00abc", + b"blob3\x00abc", + b"blob x\x00abc", + b"blob 4\x00abc", + b"blob 3", + ): + with pytest.raises(module.PolicyError, match="header is invalid"): + module.parse_loose_object(zlib.compress(header)) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.parse_loose_object(b"12345") \ No newline at end of file diff --git a/testing/tests/test_hermes_receive_pack_scan.py b/testing/tests/test_hermes_receive_pack_scan.py new file mode 100644 index 00000000..7d17fa6f --- /dev/null +++ b/testing/tests/test_hermes_receive_pack_scan.py @@ -0,0 +1,217 @@ +"""Quarantine content-scanning contracts for brokered Git pushes.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _blob_pack, + _load, + _object_entry, + _pack_of, + _receive_command, +) + +ZERO = b"0" * 40 +COMMIT = b"1" * 40 +TOKEN = "runtime-sentinel" +FORMS = (b"runtime-sentinel",) + + +def _request(pack: bytes, ref: bytes = b"refs/heads/hermes/scan") -> bytes: + return _receive_command(ZERO, COMMIT, ref, pack) + + +def test_clean_push_with_real_content_passes(): + scan = _load("receive_pack_scan") + pack = _blob_pack( + b"# services/example/deployment.yaml\napiVersion: apps/v1\n", + b"tree 5b3a\nparent none\n", + ) + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +@pytest.mark.parametrize( + "payload", + [ + b"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaA==\n", + b"-----BEGIN RSA PRIVATE KEY-----\nMIIE\n", + b"key = ghp_" + b"a1" * 12, + b"aws_access_key_id = AKIAABCDEFGHIJKLMNOP", + b"jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJlLXNlZ21lbnQ", + b"hook https://hooks.slack.com/services/T000/B000/tokentokentokentoken", + b"AGE-SECRET-KEY-1QQPCXR7QQPCXR7QQPCXR7QQPCXR7", + b"PuTTY-User-Key-File-3: ssh-ed25519", + b"Authorization: Bearer abc123-tokenvalue", + b"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF2c hermes@atlas", + b"gitea gta_" + b"z9" * 12, + ], +) +def test_credential_shaped_object_content_is_rejected(payload): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.validate_receive_pack(_request(_blob_pack(payload)), TOKEN, FORMS) + + +def test_runtime_token_hidden_by_compression_is_still_caught(): + scan = _load("receive_pack_scan") + pack = _blob_pack(b"config value: runtime-sentinel\n") + # The compressed wire bytes do not contain the token; only the + # decompressed payload does. + assert b"runtime-sentinel" not in pack + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _typed_header(type_code: int, size: int) -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + + +def test_secret_assembled_only_by_delta_resolution_is_caught(): + scan = _load("receive_pack_scan") + base = b"docs: AKIAABCD" + completion = b"EFGHIJKLMNOP" + delta = ( + _size_varint(len(base)) + + _size_varint(len(base) + len(completion)) + + bytes([0x80 | 0x10, len(base)]) + + bytes([len(completion)]) + + completion + ) + base_sha = hashlib.sha1(b"blob %d\x00" % len(base) + base).digest() + delta_entry = _typed_header(7, len(delta)) + base_sha + zlib.compress(delta) + pack = _pack_of([_object_entry(3, base), delta_entry]) + # Neither the base nor the delta insert alone matches; the resolved + # payload is the only place the credential exists. + with pytest.raises(scan.PolicyError, match="runtime credential|credential-shaped"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def test_thin_pack_deltas_are_rejected_outright(): + scan = _load("receive_pack_scan") + delta = _size_varint(4) + _size_varint(1) + bytes([1]) + b"x" + entry = _typed_header(7, len(delta)) + b"\xbb" * 20 + zlib.compress(delta) + with pytest.raises(scan.PolicyError, match="push full packs"): + scan.validate_receive_pack(_request(_pack_of([entry])), TOKEN, FORMS) + + +def test_trailing_bytes_after_the_pack_are_rejected(): + scan = _load("receive_pack_scan") + body = _request(_pack_of([])) + b"extra" + with pytest.raises(scan.PolicyError, match="bytes after its pack"): + scan.validate_receive_pack(body, TOKEN, FORMS) + + +@pytest.mark.parametrize( + ("line", "match"), + [ + (b"shallow " + COMMIT, "ref command is invalid"), + (COMMIT + b" " + COMMIT + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + ZERO + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + COMMIT + b" refs/heads/main", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/tags/v1", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/heads/hermes/\xff", "canonical ASCII"), + (b"zz" * 20 + b" " + COMMIT + b" refs/heads/hermes/x", "ref command is invalid"), + ], +) +def test_push_commands_outside_the_narrow_grant_are_rejected(line, match): + scan = _load("receive_pack_scan") + framed = f"{len(line) + 5:04x}".encode() + line + b"\n" + b"0000" + with pytest.raises(scan.PolicyError, match=match): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_command_stream_bounds_are_enforced(): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="no ref command"): + scan.validate_receive_pack(b"0000" + _pack_of([]), TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="omitted its command terminator"): + scan.validate_receive_pack(b"", TOKEN, FORMS) + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x\n" + framed = f"{len(line) + 4:04x}".encode() + line + many = framed * (scan.MAX_PUSH_COMMANDS + 1) + b"0000" + _pack_of([]) + with pytest.raises(scan.PolicyError, match="too many ref commands"): + scan.validate_receive_pack(many, TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"zzzz", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0003", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0009abc", TOKEN, FORMS) + + +def test_credential_forms_in_command_lines_are_rejected(): + scan = _load("receive_pack_scan") + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x runtime-sentinel\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_scan_crosses_chunk_boundaries_with_carry(monkeypatch): + scan = _load("receive_pack_scan") + monkeypatch.setattr(scan, "SCAN_CHUNK", 8) + monkeypatch.setattr(scan, "SCAN_CARRY", 48) + payload = b"x" * 20 + b" AKIAABCDEFGHIJKLMNOP " + b"y" * 20 + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan._scan_payload(io.BytesIO(payload), FORMS) + scan._scan_payload(io.BytesIO(b"clean " * 40), FORMS) + scan._scan_payload(io.BytesIO(b""), FORMS) + + +def test_scan_loose_object_reports_type_and_scans_content(): + scan = _load("receive_pack_scan") + clean = zlib.compress(b"blob 5\x00clean") + assert scan.scan_loose_object(clean, FORMS) == 3 + secret = b"-----BEGIN EC PRIVATE KEY-----" + dirty = zlib.compress(b"blob %d\x00" % len(secret) + secret) + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.scan_loose_object(dirty, FORMS) + leaked = b"the runtime-sentinel value" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.scan_loose_object( + zlib.compress(b"blob %d\x00" % len(leaked) + leaked), FORMS + ) + + +def test_content_patterns_extend_the_shared_standalone_corpus(): + scan = _load("receive_pack_scan") + policy = _load("gitea_api_policy") + assert len(scan.CONTENT_SECRET_PATTERNS) == len(policy.STANDALONE_SECRET_PATTERNS) + 3 + assert all( + isinstance(pattern.pattern, bytes) for pattern in scan.CONTENT_SECRET_PATTERNS + ) + + +def test_validation_rewinds_the_stream_for_upstream_forwarding(): + scan = _load("receive_pack_scan") + body = _request(_blob_pack(b"safe content")) + stream = io.BytesIO(body) + scan.validate_receive_pack(stream, TOKEN, FORMS) + assert stream.tell() == 0 + with pytest.raises(scan.PolicyError): + scan.validate_receive_pack(io.BytesIO(b"zzzz"), TOKEN, FORMS) \ No newline at end of file diff --git a/testing/tests/test_hermes_scm_broker_internal_coverage.py b/testing/tests/test_hermes_scm_broker_internal_coverage.py index abab8406..b38702ac 100644 --- a/testing/tests/test_hermes_scm_broker_internal_coverage.py +++ b/testing/tests/test_hermes_scm_broker_internal_coverage.py @@ -7,7 +7,11 @@ from email.message import Message import pytest -from testing.tests.test_hermes_scm_broker_support import Response, _load +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) def test_broker_redirect_status_and_bounded_read_helpers(monkeypatch): @@ -139,18 +143,22 @@ def test_content_length_and_credential_helpers_cover_safe_and_rejected_values(): assert broker._content_length(headers, 1) == 0 forms = broker._credential_forms("sentinel") assert len(forms) == 3 and forms[0] == b"sentinel" - broker._reject_credential_bytes(b"safe", "sentinel", "response") + zero = b"0" * 40 + commit = b"1" * 40 + line = zero + b" " + commit + b" refs/heads/hermes/x " + forms[2] + b"\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" with pytest.raises(broker.PolicyError, match="credential material"): - broker._reject_credential_bytes(b"prefix " + forms[2], "sentinel", "response") + broker._validate_receive_pack(framed, "sentinel") -def test_receive_prefix_preserves_stream_position_and_accepts_bytes(): +def test_receive_pack_validation_rewinds_stream_and_accepts_bytes(): broker = _load("scm_broker") - assert broker._receive_prefix(b"bytes") == b"bytes" - stream = io.BytesIO(b"prefix") - stream.seek(3) - assert broker._receive_prefix(stream) == b"prefix" - assert stream.tell() == 3 + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/rewind") + stream = io.BytesIO(body) + stream.seek(9) + broker._validate_receive_pack(stream, "sentinel") + assert stream.tell() == 0 + broker._validate_receive_pack(body, "sentinel") @pytest.mark.parametrize( @@ -188,7 +196,8 @@ def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) with pytest.raises(broker.PolicyError, match="terminator"): broker._validate_receive_pack(command, "sentinel") - many = command * (broker.MAX_PUSH_COMMANDS + 1) + b"0000" + limit = _load("receive_pack_scan").MAX_PUSH_COMMANDS + many = command * (limit + 1) + b"0000" with pytest.raises(broker.PolicyError, match="too many"): broker._validate_receive_pack(many, "sentinel") diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py index 2fa9683f..78a63f27 100644 --- a/testing/tests/test_hermes_scm_broker_streaming.py +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -130,7 +130,7 @@ def test_upstream_stream_applies_absolute_socket_deadline(): sock = DeadlineSocket() response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() spool, length = broker._spool_response( - response, broker.MAX_GIT_RESPONSE, "sentinel" + response, broker.MAX_GIT_RESPONSE, "sentinel", broker.INBOUND_BODY_TIMEOUT ) try: assert length == 4 diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py index 6e2184d9..e3682494 100644 --- a/testing/tests/test_hermes_scm_broker_support.py +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -2,9 +2,11 @@ from __future__ import annotations +import hashlib import importlib.util import io import sys +import zlib from email.message import Message from pathlib import Path @@ -50,6 +52,33 @@ class Response: return self.stream.read(limit) -def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes: +def _object_entry(type_code: int, payload: bytes) -> bytes: + """Encode one non-delta pack entry with a real header and zlib body.""" + size = len(payload) + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + zlib.compress(payload) + + +def _pack_of(entries: list[bytes]) -> bytes: + """Assemble encoded entries into a checksummed version-2 pack.""" + body = b"PACK" + (2).to_bytes(4, "big") + len(entries).to_bytes(4, "big") + for entry in entries: + body += entry + return body + hashlib.sha1(body).digest() + + +def _blob_pack(*payloads: bytes) -> bytes: + return _pack_of([_object_entry(3, payload) for payload in payloads]) + + +def _receive_command(old: bytes, new: bytes, ref: bytes, pack: bytes = b"") -> bytes: command = old + b" " + new + b" " + ref + b"\x00report-status\n" - return f"{len(command) + 4:04x}".encode() + command + b"0000PACK" + framed = f"{len(command) + 4:04x}".encode() + command + b"0000" + return framed + (pack or _pack_of([])) From b8f2163af0da2eda5cef68825244a8bad88a3109 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 32/46] hermes: prove sudo and polkit denial closed Expand sudoers User_Alias chains so aliases, wildcards, netgroups, and undefined names cannot smuggle authority to the Hermes account, and require polkit grants to scope through exact literal identity comparisons: computed strings, bracket lookups, subject aliasing, operator-built values, and unconditional or wildcard grants fail closed. Co-Authored-By: Claude Fable 5 --- services/hermes/kustomization.yaml | 1 + services/hermes/scripts/node_account_audit.py | 88 +++++++- services/hermes/scripts/node_polkit_audit.py | 161 +++++++++++++ .../tests/test_hermes_node_polkit_identity.py | 211 ++++++++++++++++++ .../test_hermes_node_sudo_alias_audit.py | 134 +++++++++++ 5 files changed, 591 insertions(+), 4 deletions(-) create mode 100644 services/hermes/scripts/node_polkit_audit.py create mode 100644 testing/tests/test_hermes_node_polkit_identity.py create mode 100644 testing/tests/test_hermes_node_sudo_alias_audit.py diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index fe60bef9..fc39cc66 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -108,6 +108,7 @@ configMapGenerator: - node_account_hardening.py=scripts/node_account_hardening.py - node_account_audit.py=scripts/node_account_audit.py - node_account_io.py=scripts/node_account_io.py + - node_polkit_audit.py=scripts/node_polkit_audit.py options: disableNameSuffixHash: true - name: hermes-auto-router-plugin diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py index b1795e99..31b54302 100644 --- a/services/hermes/scripts/node_account_audit.py +++ b/services/hermes/scripts/node_account_audit.py @@ -9,6 +9,7 @@ import struct from pathlib import Path from node_account_io import HardeningError, read_regular +from node_polkit_audit import audit_polkit_policy def _members(field: str, context: str) -> set[str]: @@ -108,6 +109,87 @@ def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> b return bool(re.search(account_pattern, text) or re.search(numeric_pattern, text)) +SUDO_ALIAS_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*\Z") + + +def _joined_sudo_lines(active: str) -> list[str]: + """Merge sudoers backslash continuations into whole logical lines.""" + lines: list[str] = [] + pending = "" + for line in active.splitlines(): + if line.endswith("\\"): + pending += line[:-1] + " " + continue + lines.append(pending + line) + pending = "" + if pending: + raise HardeningError("sudo policy ends inside a line continuation") + return lines + + +def _sudo_user_aliases(lines: list[str]) -> dict[str, list[str]]: + """Collect User_Alias definitions so grant principals can be expanded.""" + aliases: dict[str, list[str]] = {} + for line in lines: + stripped = line.strip() + if not stripped.startswith("User_Alias"): + continue + for definition in stripped.removeprefix("User_Alias").split(":"): + name, separator, members = definition.partition("=") + name = name.strip() + values = [item.strip() for item in members.split(",")] + if ( + not separator + or not SUDO_ALIAS_NAME_RE.fullmatch(name) + or name in aliases + or not all(values) + ): + raise HardeningError("sudo alias definition is not auditable") + aliases[name] = values + return aliases + + +def _expand_sudo_principal( + principal: str, aliases: dict[str, list[str]], seen: frozenset[str] +) -> None: + """Reject grant principals that reach Hermes through aliases or wildcards.""" + value = principal.strip() + while value.startswith("!"): + value = value[1:].strip() + value = value.strip('"') + if not value: + raise HardeningError("sudo grant principal is malformed") + if value in aliases: + if value in seen: + raise HardeningError("sudo alias expansion is cyclic") + for member in aliases[value]: + _expand_sudo_principal(member, aliases, seen | {value}) + return + if value in {"ALL", "%ALL"}: + raise HardeningError("broad sudo authority includes Hermes") + if value.startswith("+") or value.startswith("%:"): + raise HardeningError("sudo netgroup authority cannot be audited") + if SUDO_ALIAS_NAME_RE.fullmatch(value): + raise HardeningError("sudo alias is undefined") + + +def _audit_sudo_grants(active: str) -> None: + """Expand every grant's user list; only literal non-Hermes principals pass.""" + lines = _joined_sudo_lines(active) + aliases = _sudo_user_aliases(lines) + skip = ("Defaults", "User_Alias", "Runas_Alias", "Host_Alias", "Cmnd_Alias") + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith(skip): + continue + head, separator, _rest = stripped.partition("=") + fields = head.split() + if not separator or len(fields) < 2: + raise HardeningError("sudo grant line is not auditable") + for principal in " ".join(fields[:-1]).split(","): + _expand_sudo_principal(principal, aliases, frozenset()) + + def audit_privilege_policies( account: str, account_uid: int, @@ -160,10 +242,7 @@ def audit_privilege_policies( active = _active_sudo_policy(text) if _mentions_dedicated_identity(active, account, account_uid): raise HardeningError("dedicated Hermes account has sudo authority") - for line in active.splitlines(): - fields = line.split() - if fields and fields[0] in {"ALL", "%ALL"} and "=" in line: - raise HardeningError("broad sudo authority includes Hermes") + _audit_sudo_grants(active) continue active = "\n".join( line for line in text.splitlines() if not line.lstrip().startswith("#") @@ -176,3 +255,4 @@ def audit_privilege_policies( broad = "unix-user:*" in active or "Identity=unix-user:*" in active if broad and grants and any(item in active for item in dangerous_polkit): raise HardeningError("root-equivalent broad polkit authority includes Hermes") + audit_polkit_policy(path, text) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py new file mode 100644 index 00000000..281f1484 --- /dev/null +++ b/services/hermes/scripts/node_polkit_audit.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Literal-identity proof for polkit policies on hardened Atlas nodes. + +Polkit rules are JavaScript, so a grant can name its subject indirectly: +concatenated strings, escape sequences, bracket lookups, or arithmetic can +compute an identity that plain text matching never sees. Rather than model +the language, every grant must scope itself through exact literal identity +comparisons. Anything the audit cannot prove literal fails closed. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from node_account_io import HardeningError + +_ALLOWED_ESCAPES = {'"', "'", "\\", "/", "n", "r", "t"} +_STRING_MARK = r"\x00[0-9]+\x00" +_GRANT_RE = re.compile(r"\bResult\s*\.\s*YES\b") +_PKLA_GRANT_RE = re.compile(r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$") +_PKLA_IDENTITY_RE = re.compile(r"unix-(?:user|group):[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +_IDENTITY_USE_RE = re.compile(r"subject\s*(\[|\.\s*(?:user|uid|isInGroup)\b)") +_COMPUTED_TOKEN_RE = re.compile( + r"\b(?:eval|Function|arguments|fromCharCode|fromCodePoint|atob|btoa|unescape" + r"|escape|decodeURI[A-Za-z]*|encodeURI[A-Za-z]*|charCodeAt|codePointAt" + r"|normalize|constructor|prototype|globalThis|Reflect|Proxy)\b" +) +# Logical && and || stay legal; every value-building operator next to a +# string or number literal is rejected so comparisons stay whole literals. +_OPERATOR_ADJACENT_RE = re.compile( + rf"(?:{_STRING_MARK}|[0-9])\s*[+\-*/%<>^~](?![&|=])" + rf"|(?])[+\-*/%<>^~]\s*(?:{_STRING_MARK}|[0-9])" +) +_USER_BEFORE_RE = re.compile(rf"{_STRING_MARK}\s*[=!]==?\s*\Z") +_USER_AFTER_RE = re.compile(rf"\s*[=!]==?\s*{_STRING_MARK}") +_UID_BEFORE_RE = re.compile(r"(? tuple[str, list[str]]: + """Replace string literals with markers and drop JavaScript comments.""" + code: list[str] = [] + strings: list[str] = [] + index = 0 + length = len(text) + while index < length: + character = text[index] + if character == "`": + raise HardeningError("polkit rule uses computed template strings") + if character in {'"', "'"}: + quote = character + index += 1 + value: list[str] = [] + while True: + if index >= length or text[index] in "\r\n": + raise HardeningError("polkit rule string is unterminated") + character = text[index] + if character == "\\": + if index + 1 >= length or text[index + 1] not in _ALLOWED_ESCAPES: + raise HardeningError("polkit rule string uses computed escapes") + escaped = text[index + 1] + value.append({"n": "\n", "r": "\r", "t": "\t"}.get(escaped, escaped)) + index += 2 + continue + index += 1 + if character == quote: + break + value.append(character) + strings.append("".join(value)) + code.append(f"\x00{len(strings) - 1}\x00") + continue + if text.startswith("//", index): + newline = text.find("\n", index) + index = length if newline < 0 else newline + continue + if text.startswith("/*", index): + closing = text.find("*/", index + 2) + if closing < 0: + raise HardeningError("polkit rule comment is unterminated") + code.append(" ") + index = closing + 2 + continue + code.append(character) + index += 1 + return "".join(code), strings + + +def _require_literal_identity_use(code: str, match: re.Match[str]) -> None: + """Reject one identity reference unless it is an exact literal comparison.""" + form = match.group(1) + if form.startswith("["): + raise HardeningError("polkit rule reads identity through a computed lookup") + member = form.removeprefix(".").strip() + before = code[max(match.start() - 48, 0) : match.start()] + after = code[match.end() : match.end() + 48] + if member == "user": + literal = _USER_BEFORE_RE.search(before) or _USER_AFTER_RE.match(after) + elif member == "uid": + literal = _UID_BEFORE_RE.search(before) or _UID_AFTER_RE.match(after) + else: + literal = _GROUP_AFTER_RE.match(after) + if not literal: + raise HardeningError("polkit rule identity test is not an exact literal") + + +def _audit_rules_javascript(text: str) -> None: + """Require granting rules to gate on exact literal subject identities. + + With computed tokens, bracket lookups, and subject aliasing rejected in + every rule file, a YES decision can only be written as the literal grant + token, so the grant test below cannot be evaded by construction. + """ + code, _strings = _split_strings(text) + if _COMPUTED_TOKEN_RE.search(code): + raise HardeningError("polkit rule computes values the audit cannot prove") + if _RESULT_USE_RE.search(code): + raise HardeningError("polkit rule reads results through a computed lookup") + for bare in _BARE_SUBJECT_RE.finditer(code): + if not _PARAMETER_BEFORE_RE.search(code[: bare.start()]): + raise HardeningError("polkit rule passes the subject to computed code") + if not _GRANT_RE.search(code): + return + if _OPERATOR_ADJACENT_RE.search(code): + raise HardeningError("polkit grant builds values with operators") + uses = 0 + for match in _IDENTITY_USE_RE.finditer(code): + uses += 1 + _require_literal_identity_use(code, match) + if not uses: + raise HardeningError("polkit grant is not scoped to an explicit identity") + + +def _audit_pkla(text: str) -> None: + """Require granting localauthority entries to name exact literal identities.""" + if not _PKLA_GRANT_RE.search(text): + return + identities: list[str] = [] + for line in text.splitlines(): + name, separator, value = line.strip().partition("=") + if name.strip() != "Identity" or not separator: + continue + entries = [entry.strip() for entry in value.split(";") if entry.strip()] + identities.extend(entries) + if not identities: + raise HardeningError("polkit authority grant lacks an identity scope") + for entry in identities: + if not _PKLA_IDENTITY_RE.fullmatch(entry): + raise HardeningError("polkit authority identity is not an exact literal") + + +def audit_polkit_policy(path: Path, text: str) -> None: + """Prove one polkit policy file cannot grant through a computed identity.""" + if "localauthority" in path.parts or path.suffix == ".pkla": + _audit_pkla(text) + else: + _audit_rules_javascript(text) diff --git a/testing/tests/test_hermes_node_polkit_identity.py b/testing/tests/test_hermes_node_polkit_identity.py new file mode 100644 index 00000000..059f05f2 --- /dev/null +++ b/testing/tests/test_hermes_node_polkit_identity.py @@ -0,0 +1,211 @@ +"""Literal-identity denial proofs for polkit grants near the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_polkit_audit_test", SCRIPTS / "node_polkit_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _rules(module, text: str): + module.audit_polkit_policy(Path("rules.d/50-test.rules"), text) + + +def test_literal_distribution_rule_passes(): + module = _load() + _rules( + module, + "// allow the network daemon\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.network1.set-dns" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + _rules( + module, + "polkit.addRule(function(action, subject) {\n" + ' if (subject.isInGroup("wheel") && subject.uid === 4242) {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + _rules( + module, + "/* deny-only rules may use constructs the audit cannot prove */\n" + "polkit.addRule(function(action, subject) {\n" + " if (subject.user.indexOf(\"x\") == 0) { return polkit.Result.NO; }\n" + "});\n", + ) + + +def test_reversed_literal_comparisons_pass(): + module = _load() + _rules( + module, + "polkit.addRule(function(action, subject) {\n" + ' if ("atlas" == subject.user || 4242 === subject.uid) {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + +def test_unconditional_grant_fails_closed(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result.YES; });\n") + + +@pytest.mark.parametrize( + ("snippet", "match"), + [ + ('subject.user == "herme" + "s-agent"', "operators"), + ("subject.uid == 1199 + 1", "operators"), + ("subject.uid == 0x4B0", "exact literal"), + ("subject.uid >= 1000", "exact literal"), + ("subject.uid == -1", "operators"), + ("subject.user == name", "exact literal"), + ("subject.user.indexOf(\"h\") == 0", "exact literal"), + ("subject.isInGroup(group)", "exact literal"), + ('subject["user"] == "atlas"', "computed lookup"), + ], +) +def test_computed_identity_grants_fail_closed(snippet: str, match: str): + module = _load() + text = ( + "polkit.addRule(function(action, subject) {\n" + f" if ({snippet}) {{ return polkit.Result.YES; }}\n" + "});\n" + ) + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ("var x = `hermes`;\n", "template strings"), + ('var x = "unterminated\n', "unterminated"), + ('var x = "bad\\x2descape";\n', "computed escapes"), + ('var x = "dangling\\', "computed escapes"), + ("/* never closed\n", "comment is unterminated"), + ("var f = eval;\n", "cannot prove"), + ('var s = String.fromCharCode(104);\n', "cannot prove"), + ("var a = arguments;\n", "cannot prove"), + ('var r = polkit.Result["YES"];\n', "computed lookup"), + ("var r = polkit.Result;\n", "computed lookup"), + ("var s = subject;\n", "computed code"), + ("check(subject);\n", "computed code"), + ], +) +def test_unprovable_rule_constructs_fail_closed(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +def test_string_and_comment_handling_keeps_literal_rules_auditable(): + module = _load() + code, strings = module._split_strings( + 'var a = "with // not a comment"; // trailing\n' + "/* block */ var b = 'quo\\'te\\n';\n" + ) + assert strings == ["with // not a comment", "quo'te\n"] + assert "//" not in code and "block" not in code + assert "\x000\x00" in code and "\x001\x00" in code + + +def test_grant_detection_survives_spacing_and_strings(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result . YES; });\n") + # A grant token inside a string is inert and needs no identity scope. + _rules(module, 'var label = "polkit.Result.YES";\n') + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ( + "[All]\nIdentity=unix-user:*\nAction=org.example\nResultAny=yes\n", + "exact literal", + ), + ( + "[N]\nIdentity=unix-netgroup:ops\nAction=org.example\nResultActive=yes\n", + "exact literal", + ), + ("[N]\nAction=org.example\nResultInactive=yes\n", "identity scope"), + ("[N]\nIdentity=\nAction=org.example\nResultAny=yes\n", "identity scope"), + ], +) +def test_pkla_grants_require_exact_literal_identities(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + module.audit_polkit_policy(Path("localauthority/50-local.d/a.pkla"), text) + + +def test_pkla_literal_grants_and_deny_entries_pass(): + module = _load() + module.audit_polkit_policy( + Path("localauthority/50-local.d/a.pkla"), + "[Mount]\nIdentity=unix-user:atlas;unix-group:operators\n" + "Action=org.freedesktop.udisks2.filesystem-mount\nResultAny=yes\n", + ) + module.audit_polkit_policy( + Path("etc/b.pkla"), + "[Deny]\nIdentity=unix-user:*\nAction=org.example\nResultAny=no\n", + ) + + +def test_integration_rejects_previously_invisible_grants(tmp_path: Path): + spec = importlib.util.spec_from_file_location( + "node_account_audit_polkit_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + audit = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = audit + spec.loader.exec_module(audit) + + host_etc = tmp_path / "etc" + rules = host_etc / "polkit-1/rules.d" + rules.mkdir(parents=True) + (rules / "90-open.rules").write_text( + "polkit.addRule(function() { return polkit.Result.YES; });\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="explicit identity"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) + + (rules / "90-open.rules").unlink() + local = host_etc / "polkit-1/localauthority/50-local.d" + local.mkdir(parents=True) + (local / "wide.pkla").write_text( + "[W]\nIdentity=unix-user:*\nAction=org.example.benign\nResultAny=yes\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="exact literal"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) \ No newline at end of file diff --git a/testing/tests/test_hermes_node_sudo_alias_audit.py b/testing/tests/test_hermes_node_sudo_alias_audit.py new file mode 100644 index 00000000..e62b82bf --- /dev/null +++ b/testing/tests/test_hermes_node_sudo_alias_audit.py @@ -0,0 +1,134 @@ +"""Sudo alias and computed-identity denial proofs for the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_account_audit_sudo_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _audit(tmp_path: Path, sudoers: str): + module = _load() + host_etc = tmp_path / "etc" + (host_etc / "sudoers.d").mkdir(parents=True, exist_ok=True) + (host_etc / "sudoers.d/atlas").write_text(sudoers, encoding="utf-8") + module.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing-share" + ) + + +def test_benign_distribution_policy_passes(tmp_path: Path): + _audit( + tmp_path, + "Defaults secure_path=/usr/bin\n" + "root ALL=(ALL:ALL) ALL\n" + "%sudo ALL=(ALL:ALL) ALL\n" + "#999 ALL=(root) /usr/bin/uptime\n" + "alice, bob ALL=(root) NOPASSWD: /usr/bin/systemctl status\n", + ) + + +def test_alias_reaching_hermes_directly_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit( + tmp_path, + "User_Alias OPERATORS = atlas, hermes-agent\n" + "OPERATORS ALL=(ALL) ALL\n", + ) + + +def test_alias_reaching_hermes_by_numeric_uid_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit(tmp_path, "User_Alias OPERATORS = #1200\nOPERATORS ALL=(ALL) ALL\n") + + +def test_alias_expanding_to_all_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, "User_Alias ADMINS = ALL\nADMINS ALL=(ALL) ALL\n") + + +def test_nested_alias_expanding_to_all_via_continuation_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit( + tmp_path, + "User_Alias INNER = atlas, \\\n ALL\n" + "User_Alias OUTER = INNER\n" + "OUTER ALL=(ALL) ALL\n", + ) + + +@pytest.mark.parametrize( + ("sudoers", "match"), + [ + ("+operators ALL=(ALL) ALL\n", "netgroup"), + ("%:S-1-5-32 ALL=(ALL) ALL\n", "netgroup"), + ("GHOSTS ALL=(ALL) ALL\n", "alias is undefined"), + ("User_Alias A = B\nUser_Alias B = A\nA ALL=(ALL) ALL\n", "cyclic"), + ("User_Alias broken\nroot ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias lower = atlas\n", "not auditable"), + ("User_Alias A = atlas\nUser_Alias A = bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias A = atlas,,bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("stray-line-without-equals\n", "not auditable"), + ("=orphan (ALL) ALL\n", "not auditable"), + ("! ALL=(ALL) ALL\n", "malformed"), + ("User_Alias A = atlas : B = ALL\nB ALL=(ALL) ALL\n", "broad sudo"), + ], +) +def test_unauditable_or_broad_sudo_policies_fail_closed( + tmp_path: Path, sudoers: str, match: str +): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _audit(tmp_path, sudoers) + + +def test_negated_and_quoted_principals_are_expanded_before_judging(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, '!"ALL" ALL=(ALL) ALL\n') + _audit(tmp_path, "!alice ALL=(ALL) ALL\n") + + +def test_continuation_join_requires_a_complete_final_line(): + module = _load() + with pytest.raises(module.HardeningError, match="line continuation"): + module._joined_sudo_lines("root ALL=(ALL) ALL \\") + assert module._joined_sudo_lines("a \\\nb\nc") == ["a b", "c"] + + +def test_alias_definitions_parse_multiple_groups_per_line(): + module = _load() + aliases = module._sudo_user_aliases( + ["User_Alias A = atlas, bob : B = carol", "Runas_Alias R = root"] + ) + assert aliases == {"A": ["atlas", "bob"], "B": ["carol"]} + + +def test_expansion_accepts_nested_literal_users(): + module = _load() + aliases = {"A": ["atlas", "B"], "B": ["carol"]} + module._expand_sudo_principal("A", aliases, frozenset()) + module._expand_sudo_principal("%wheel", aliases, frozenset()) + module._expand_sudo_principal("#999", aliases, frozenset()) \ No newline at end of file From 5020e49057c911469c04a002b97b1329d8b8266a Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 33/46] hermes: track new boundary modules in gate Co-Authored-By: Claude Fable 5 --- testing/quality_contract.json | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 805857d9..78fa2f6a 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -40,11 +40,15 @@ "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", - "testing/tests/test_publish_test_metrics.py", - "testing/tests/test_supply_chain_report.py", - "testing/tests/test_semgrep_report.py", - "testing/tests/test_quality_contract.py", - "testing/tests/test_quality_gate.py" + "testing/tests/test_publish_test_metrics.py", + "testing/tests/test_supply_chain_report.py", + "testing/tests/test_semgrep_report.py", + "testing/tests/test_quality_contract.py", + "testing/tests/test_quality_gate.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "lint_paths": [ "ci/scripts/publish_test_metrics.py", @@ -63,7 +67,8 @@ "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", "testing/tests", - "testing" + "testing", + "services/hermes/scripts/node_polkit_audit.py" ], "pytest_suites": { "unit": { @@ -154,7 +159,8 @@ "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", "services/mailu/scripts/mailu_sync.py", - "services/mailu/scripts/mailu_sync_listener.py" + "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/node_polkit_audit.py" ], "naming_rules": [ { @@ -210,7 +216,11 @@ "services/hermes/scripts/node_account_hardening.py", "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", - "testing/quality_coverage.py" + "testing/quality_coverage.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "tracked_files": [ "ci/scripts/publish_test_metrics.py", @@ -235,7 +245,11 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", - "testing/quality_gate.py" + "testing/quality_gate.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ] } } From 6f993c049e4def6014a84ee6970f96f326d2ba78 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:20:43 -0300 Subject: [PATCH 34/46] hermes: keep agent web reachable during lane deferral Co-Authored-By: Claude Fable 5 --- services/hermes/agent-deployment.yaml | 4 ++-- services/hermes/oauth2-proxy.yaml | 3 +++ testing/quality_contract.json | 1 - testing/tests/test_hermes_agent_layout.py | 17 +++++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 57611a51..85f563d6 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -903,8 +903,8 @@ spec: - /opt/coordinator/cli_lane_capabilities.py initialDelaySeconds: 2 periodSeconds: 5 - timeoutSeconds: 2 - failureThreshold: 1 + timeoutSeconds: 5 + failureThreshold: 3 resources: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "2", memory: 6Gi} diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml index 24383b3a..52dea326 100644 --- a/services/hermes/oauth2-proxy.yaml +++ b/services/hermes/oauth2-proxy.yaml @@ -16,6 +16,9 @@ metadata: spec: selector: app: hermes-agent + # The agent dashboard and terminal must stay reachable while the + # cli-lane-runner readiness probe holds the shared pod NotReady. + publishNotReadyAddresses: true ports: - {name: http, port: 80, targetPort: auth-http} --- diff --git a/testing/quality_contract.json b/testing/quality_contract.json index cfcc0e43..e1dc7662 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -91,7 +91,6 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", - "scripts.render.dashboards_render_atlas", "services/hermes/scripts", "services/mailu/scripts", "testing" diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 72ab7f05..4f645d4f 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -285,6 +285,19 @@ def test_broker_services_survive_sibling_container_readiness_loss(): ): assert services[name]["spec"]["publishNotReadyAddresses"] is True + # The agent web endpoints route through this Service to the same pod the + # cli-lane-runner readiness probe gates; a deferred lane image must not + # take down the dashboard or terminal. + agent_auth = next( + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + and item["kind"] == "Service" + and item["metadata"]["name"] == "oauth2-proxy-hermes-agent" + ) + assert agent_auth["spec"]["selector"] == {"app": "hermes-agent"} + assert agent_auth["spec"]["publishNotReadyAddresses"] is True + def test_cli_lane_domain_modules_are_all_mounted_with_the_runner(): manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text()) @@ -354,8 +367,8 @@ def test_cli_lane_config_refresh_does_not_restart_active_work(): }, "initialDelaySeconds": 2, "periodSeconds": 5, - "timeoutSeconds": 2, - "failureThreshold": 1, + "timeoutSeconds": 5, + "failureThreshold": 3, } environment = {item["name"]: item["value"] for item in lane["env"]} assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60" From 62d8cd984b79bbe7f831ac4d2cd32facbd05e7fe Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:30:53 -0300 Subject: [PATCH 35/46] hermes: wire quota-aware lane routing config and metrics scrape Add kanban routing keys (provider_quota_min_remaining_percent: 15, provider_capacity_cooldown_seconds: 300, provider_auth_cooldown_seconds: 3600), a lane-metrics port/Service on 9011 with service-annotation scraping, monitoring ingress for the new port, and the lane's quota metrics URL env. Based on PR #15 (fix/hermes-result-decomposition- reliability); stacked because this work rides on the decomposed lane. Co-Authored-By: Claude Opus 4.8 --- services/hermes/agent-configmap.yaml | 8 ++++++++ services/hermes/agent-deployment.yaml | 6 +++++- services/hermes/networkpolicy.yaml | 1 + services/hermes/service.yaml | 21 +++++++++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 0b81379d..cc49951c 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -88,6 +88,14 @@ data: auto_decompose: true auto_decompose_per_tick: 2 dispatch_stale_timeout_seconds: 14400 + # Steady-state quota-aware routing for the direct CLI lane: below this + # remaining-percent a provider stops receiving NEW cli-auto work while + # active work finishes; lane-observed capacity/auth failures exclude a + # provider until the cooldown elapses, its quota window resets, or a + # fresh success is recorded. + provider_quota_min_remaining_percent: 15 + provider_capacity_cooldown_seconds: 300 + provider_auth_cooldown_seconds: 3600 model_catalog: enabled: true diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 57611a51..97ec0e81 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6" + ai.bstein.dev/config-rev: "20260817-cli-auto-quota-routing-v1" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" @@ -866,6 +866,8 @@ spec: . /opt/data/.env set +a exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py + ports: + - {name: lane-metrics, containerPort: 9011, protocol: TCP} env: - {name: HERMES_HOME, value: /opt/data} - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json} @@ -876,6 +878,8 @@ spec: - {name: PYTHONPATH, value: /opt/hermes} - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"} - {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"} + - {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"} + - {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} securityContext: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index d972889e..3205230e 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -119,6 +119,7 @@ spec: app: server ports: - {protocol: TCP, port: 9010} + - {protocol: TCP, port: 9011} # agent.hermes.bstein.dev is an owner-only engineering workstation. The # browser boundary remains OAuth-protected, while its workers need to reach # every cluster namespace, Atlas LAN service, and hosted provider endpoint. diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index 0fc8d939..7e7f868e 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -150,3 +150,24 @@ spec: port: 9006 targetPort: claude-broker protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-cli-lane-metrics + namespace: hermes + labels: + app: hermes-agent + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9011" + prometheus.io/path: /metrics +spec: + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: lane-metrics + port: 9011 + targetPort: lane-metrics + protocol: TCP From 034c8372c7b31fe45c00a1229046f4b01973495b Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:31:05 -0300 Subject: [PATCH 36/46] hermes: enforce quota-aware fail-closed cli-auto provider routing Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked on the decomposed cli_lane modules. - cli_lane_quota: soft-exclude a provider from NEW cli-auto work below the remaining-quota threshold (both-below prefers more remaining; fetch failure fails open with a metric). - cli_lane_health: lane now writes provider health (G7) with classified failure reasons splitting the capacity conflation (quota/auth/ rate-limit/transport) and cooldown hysteresis; re-admission only on full cooldown expiry, passed quota reset, or fresh success (G4). - cli_lane_routing: capacity-limited health now excludes a provider (G3); cooldown/reset-aware re-admission. - cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail closed as transient instead of switching providers (G5); fallback depth stays bounded at two hosted providers (G1) with effort preserved; Switchyard outages block transient, not capability (G9). - cli_lane_metrics: route-decision/fallback counters, quota and soft-exclusion gauges, pod-local scrape server (G6). - cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY, OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10). Co-Authored-By: Claude Opus 4.8 --- services/hermes/scripts/cli_lane_config.py | 23 ++ services/hermes/scripts/cli_lane_dispatch.py | 2 + services/hermes/scripts/cli_lane_execution.py | 132 ++++++----- services/hermes/scripts/cli_lane_failover.py | 181 ++++++++++++++++ services/hermes/scripts/cli_lane_health.py | 102 +++++++++ services/hermes/scripts/cli_lane_metrics.py | 188 ++++++++++++++++ services/hermes/scripts/cli_lane_provider.py | 4 + services/hermes/scripts/cli_lane_quota.py | 205 ++++++++++++++++++ services/hermes/scripts/cli_lane_routing.py | 44 +++- services/hermes/scripts/cli_lane_runner.py | 34 +++ testing/quality_contract.json | 20 +- 11 files changed, 870 insertions(+), 65 deletions(-) create mode 100644 services/hermes/scripts/cli_lane_failover.py create mode 100644 services/hermes/scripts/cli_lane_health.py create mode 100644 services/hermes/scripts/cli_lane_metrics.py create mode 100644 services/hermes/scripts/cli_lane_quota.py diff --git a/services/hermes/scripts/cli_lane_config.py b/services/hermes/scripts/cli_lane_config.py index dcf29faa..d570b2ca 100644 --- a/services/hermes/scripts/cli_lane_config.py +++ b/services/hermes/scripts/cli_lane_config.py @@ -11,6 +11,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import yaml + import cli_lane_goal @@ -31,6 +33,12 @@ DEFAULT_MAX_RUNTIME = 12 * 60 * 60 HEARTBEAT_SECONDS = 20 PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60 +QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0 +CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0 +AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0 +QUOTA_METRICS_URL = os.environ.get( + "HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics" +) KANBAN_STORAGE_ATTEMPTS = 5 ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60 ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60 @@ -194,3 +202,18 @@ class TerminalRecoverySnapshot: def utc_now() -> str: return datetime.now(timezone.utc).isoformat() + + +def kanban_setting(name: str, default: float) -> float: + """Read one numeric routing setting from the deployed kanban config block.""" + try: + document = yaml.safe_load( + (DATA_ROOT / "config.yaml").read_text(encoding="utf-8") + ) + except (OSError, yaml.YAMLError): + return default + kanban = document.get("kanban") if isinstance(document, dict) else None + value = kanban.get(name) if isinstance(kanban, dict) else None + if isinstance(value, bool) or not isinstance(value, (int, float)): + return default + return float(value) diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 3ed56c2f..2789b316 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -24,6 +24,7 @@ from cli_lane_config import ( ) from cli_lane_execution import execute_claim from cli_lane_files import atomic_json +from cli_lane_metrics import start_metrics_server from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations from cli_lane_retention import maybe_gc_lane_artifacts @@ -155,6 +156,7 @@ def main() -> int: RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644) + start_metrics_server() capabilities = initialize_kanban_capabilities(kanban_db) recover_orphans() workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8)) diff --git a/services/hermes/scripts/cli_lane_execution.py b/services/hermes/scripts/cli_lane_execution.py index 46a3d7b0..4dafe737 100644 --- a/services/hermes/scripts/cli_lane_execution.py +++ b/services/hermes/scripts/cli_lane_execution.py @@ -15,6 +15,7 @@ from cli_lane_config import ( DEFAULT_MAX_RUNTIME, TerminalFinalizationPending, ) +from cli_lane_failover import _routed_or_blocked, capacity_failover from cli_lane_files import ( _terminal_identity, _terminal_path, @@ -23,8 +24,11 @@ from cli_lane_files import ( state_path, ) from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run +from cli_lane_health import record_provider_success +from cli_lane_metrics import record_route_decision from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts from cli_lane_provider import run_provider +from cli_lane_quota import selection_constraint from cli_lane_records import _persist_candidate, _write_terminal_record from cli_lane_recovery import _has_pending_finalization from cli_lane_routing import fresh_unavailable_provider, select_route @@ -115,21 +119,30 @@ def execute_claim(board: str, task_id: str) -> None: try: previous_route = state.get("current_route") - excluded_provider = ( - fresh_unavailable_provider() if assignee == "cli-auto" else None + constraint = selection_constraint(assignee) + unavailable_provider = ( + constraint.exclude_provider if constraint.source == "health" else None ) - unavailable_provider = excluded_provider - route = select_route( - context, - assignee, - exclude_provider=excluded_provider, - exclude_reason="is unavailable according to fresh native health" - if excluded_provider - else None, + route = _routed_or_blocked( + kanban_db, + board, + task_id, + run_id, + lambda: select_route( + context, + assignee, + exclude_provider=constraint.exclude_provider, + exclude_reason=constraint.exclude_reason, + ), ) - if excluded_provider: + if route is None: + return + for note in constraint.notes: + comment(note) + if constraint.exclude_provider: comment( - f"Provider health guard excluded {excluded_provider} before automatic routing.", + f"Provider routing guard ({constraint.source}) excluded " + f"{constraint.exclude_provider} before automatic routing.", ) comment( f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}", @@ -181,44 +194,34 @@ def execute_claim(board: str, task_id: str) -> None: goal_turn=goal_turn, ) if result.capacity_failure: - unavailable_provider = route.provider - retry_context = ( - context - + "\n\nRouting boundary: the first provider failed from capacity/authentication. " - + "Select the alternate hosted provider at an appropriate effort." + boundary = capacity_failover( + kanban_db, + board=board, + task_id=task_id, + run_id=run_id, + assignee=assignee, + comment=comment, + context=context, + workspace=workspace, + state=state, + state_file=state_file, + log_path=log_path, + heartbeat=heartbeat, + deadline=deadline, + route=route, + result=result, + candidate_file=candidate_file, + goal_turn=goal_turn, ) - alternate = "claude" if route.provider == "codex" else "codex" - fallback = select_route( - retry_context, - f"cli-{alternate}-{route.effort}", - ) - comment( - f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.", - ) - result = run_provider( - fallback, - build_prompt( - context, - workspace, - git_handoff(workspace, result.output), - ), - workspace, - state, - state_file, - log_path, - heartbeat, - max(1, int(deadline - time.monotonic())), - ) - route = fallback - if result.structured: - candidate_file = _persist_candidate( - state, - state_file, - dict(result.structured), - route=route, - returncode=result.returncode, - goal_turn=goal_turn, - ) + route = boundary.route + result = boundary.result + candidate_file = boundary.candidate_file + if boundary.unavailable_provider is not None: + unavailable_provider = boundary.unavailable_provider + if boundary.blocked: + break + if result.returncode == 0: + record_provider_success(route.provider) structured = result.structured if structured: structured = dict(structured) @@ -329,6 +332,9 @@ def execute_claim(board: str, task_id: str) -> None: "accepted worker result is durably journaled but " f"Kanban finalization is {outcome}" ) + record_route_decision( + route.provider, route.effort, route.classifier, "completed" + ) break can_continue = ( @@ -358,14 +364,25 @@ def execute_claim(board: str, task_id: str) -> None: excluded = unavailable_provider if excluded is None and assignee == "cli-auto": excluded = fresh_unavailable_provider() - next_route = select_route( - escalation_context, - assignee, - exclude_provider=excluded, - exclude_reason="is unavailable according to fresh native health" - if excluded - else None, + record_route_decision( + route.provider, route.effort, route.classifier, "goal-continued" ) + next_route = _routed_or_blocked( + kanban_db, + board, + task_id, + run_id, + lambda boundary=escalation_context, blocked=excluded: select_route( + boundary, + assignee, + exclude_provider=blocked, + exclude_reason="is unavailable according to fresh native health" + if blocked + else None, + ), + ) + if next_route is None: + break comment( f"Goal route {goal_turn}/{goal_max_turns}: " f"{next_route.provider}/{next_route.model} at {next_route.effort}; " @@ -400,6 +417,9 @@ def execute_claim(board: str, task_id: str) -> None: failure_kind = ( "transient" if result.capacity_failure else "capability" ) + record_route_decision( + route.provider, route.effort, route.classifier, f"blocked-{failure_kind}" + ) _board_call( kanban_db, board, diff --git a/services/hermes/scripts/cli_lane_failover.py b/services/hermes/scripts/cli_lane_failover.py new file mode 100644 index 00000000..0f6becdc --- /dev/null +++ b/services/hermes/scripts/cli_lane_failover.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Bounded, fail-closed cross-provider failover for one capacity boundary.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +from cli_lane_board import _board_call +from cli_lane_config import ProcessResult, Route +from cli_lane_health import classify_capacity_failure, record_provider_failure +from cli_lane_metrics import ( + record_provider_fallback, + record_route_decision, + record_router_selection_failure, +) +from cli_lane_prompt import build_prompt, git_handoff +from cli_lane_provider import run_provider +from cli_lane_records import _persist_candidate +from cli_lane_routing import select_route + + +@dataclass +class FailoverOutcome: + """Where one capacity boundary left the claim's route and result.""" + + route: Route + result: ProcessResult + candidate_file: Path | None + unavailable_provider: str | None + blocked: bool + + +def _block_transient( + kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str +) -> None: + """Park one card as retryable so it re-enters when providers recover.""" + _board_call( + kanban_db, + board, + lambda fresh: kanban_db.block_task( + fresh, + task_id, + reason=reason, + kind="transient", + expected_run_id=run_id, + ), + ) + + +def _routed_or_blocked( + kanban_db: Any, + board: str, + task_id: str, + run_id: Any, + select: Callable[[], Route], +) -> Route | None: + """Turn a Switchyard outage into a transient block, never a capability one.""" + try: + return select() + except RuntimeError as error: + record_router_selection_failure() + _block_transient( + kanban_db, + board, + task_id, + run_id, + "Switchyard route selection is unavailable; the card retries " + f"when the router recovers: {error}", + ) + return None + + +def capacity_failover( + kanban_db: Any, + *, + board: str, + task_id: str, + run_id: Any, + assignee: str, + comment: Callable[[str], None], + context: str, + workspace: Path, + state: dict[str, Any], + state_file: Path, + log_path: Path, + heartbeat: Callable[[str], bool], + deadline: float, + route: Route, + result: ProcessResult, + candidate_file: Path | None, + goal_turn: int, +) -> FailoverOutcome: + """Apply the capacity policy once: record, fail closed, or fail over.""" + failure_class = classify_capacity_failure(result.output) + record_provider_failure(route.provider, failure_class) + record_route_decision( + route.provider, route.effort, route.classifier, "capacity-failure" + ) + if assignee != "cli-auto": + # Fail closed: an operator pinned this provider, so a capacity or + # auth failure must wait for that provider, never switch away. + _block_transient( + kanban_db, + board, + task_id, + run_id, + f"Manually pinned provider {route.provider} hit a " + f"{failure_class} failure; the explicit {assignee} lane never " + "switches providers. Retry after recovery or reassign to cli-auto.", + ) + return FailoverOutcome(route, result, candidate_file, None, True) + retry_context = ( + context + + "\n\nRouting boundary: the first provider failed from capacity/authentication. " + + "Select the alternate hosted provider at an appropriate effort." + ) + alternate = "claude" if route.provider == "codex" else "codex" + fallback = _routed_or_blocked( + kanban_db, + board, + task_id, + run_id, + lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"), + ) + if fallback is None: + return FailoverOutcome(route, result, candidate_file, route.provider, True) + record_provider_fallback(route.provider, fallback.provider, failure_class) + comment( + f"Provider fallback: {route.provider} -> {fallback.provider} " + f"after a {failure_class} failure; Jetson reclassified the retry boundary.", + ) + fallback_result = run_provider( + fallback, + build_prompt( + context, + workspace, + git_handoff(workspace, result.output), + ), + workspace, + state, + state_file, + log_path, + heartbeat, + max(1, int(deadline - time.monotonic())), + ) + if fallback_result.capacity_failure: + # Bounded failover depth: two hosted providers, never a third + # metered or local path. + fallback_class = classify_capacity_failure(fallback_result.output) + record_provider_failure(fallback.provider, fallback_class) + record_route_decision( + fallback.provider, fallback.effort, fallback.classifier, "capacity-failure" + ) + _block_transient( + kanban_db, + board, + task_id, + run_id, + "Both hosted providers failed at this boundary " + f"({route.provider}: {failure_class}; {fallback.provider}: " + f"{fallback_class}); the lane never falls to a metered or " + "local path. Waiting for provider recovery.", + ) + return FailoverOutcome( + fallback, fallback_result, candidate_file, route.provider, True + ) + if fallback_result.structured: + candidate_file = _persist_candidate( + state, + state_file, + dict(fallback_result.structured), + route=fallback, + returncode=fallback_result.returncode, + goal_turn=goal_turn, + ) + return FailoverOutcome( + fallback, fallback_result, candidate_file, route.provider, False + ) diff --git a/services/hermes/scripts/cli_lane_health.py b/services/hermes/scripts/cli_lane_health.py new file mode 100644 index 00000000..b9df10a0 --- /dev/null +++ b/services/hermes/scripts/cli_lane_health.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Lane-observed provider health with failure classification and cooldowns.""" + +from __future__ import annotations + +import json +import os +import re +import time +from typing import Any + +from cli_lane_config import ( + AUTH_COOLDOWN_SECONDS_DEFAULT, + CAPACITY_COOLDOWN_SECONDS_DEFAULT, + PROVIDER_HEALTH_PATHS, + kanban_setting, + utc_now, +) +from cli_lane_files import load_json + + +# Ordered from most to least specific so one output maps to one actionable +# reason instead of the single conflated capacity signal. +FAILURE_CLASSIFICATIONS = ( + ("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)), + ("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)), + ("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)), +) + + +def classify_capacity_failure(output: str) -> str: + """Split the broad capacity regex into quota/auth/rate-limit/transport.""" + for reason, pattern in FAILURE_CLASSIFICATIONS: + if pattern.search(output): + return reason + return "transport" + + +def _write_provider_health( + provider: str, updates: dict[str, Any], clear: tuple[str, ...] = () +) -> None: + """Merge one lane observation into the shared provider health snapshot.""" + path = PROVIDER_HEALTH_PATHS[provider] + value = load_json(path) + for key in clear: + value.pop(key, None) + value.update(updates) + try: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + except OSError: + # Health snapshots are advisory; never fail the lane over one write. + pass + + +def record_provider_success(provider: str) -> None: + """Publish fresh evidence of success so the provider re-enters routing.""" + _write_provider_health( + provider, + { + "source": "cli-lane-runner", + "state": "available", + "authenticated": True, + "checked_at": utc_now(), + }, + clear=("failure_reason", "failed_at", "cooldown_until"), + ) + + +def record_provider_failure( + provider: str, reason: str, now: float | None = None +) -> float: + """Persist one lane-observed failure with its re-admission cooldown.""" + current = time.time() if now is None else now + if reason == "auth": + cooldown = kanban_setting( + "provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT + ) + state, authenticated = "unavailable", False + else: + cooldown = kanban_setting( + "provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT + ) + state, authenticated = "capacity-limited", True + cooldown_until = current + max(0.0, cooldown) + _write_provider_health( + provider, + { + "source": "cli-lane-runner", + "state": state, + "authenticated": authenticated, + "checked_at": utc_now(), + "failure_reason": reason, + "failed_at": current, + "cooldown_until": cooldown_until, + }, + ) + return cooldown_until diff --git a/services/hermes/scripts/cli_lane_metrics.py b/services/hermes/scripts/cli_lane_metrics.py new file mode 100644 index 00000000..e3192998 --- /dev/null +++ b/services/hermes/scripts/cli_lane_metrics.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""In-process Prometheus metrics for the direct CLI lane's routing decisions.""" + +from __future__ import annotations + +import os +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +METRIC_HELP = { + "hermes_cli_route_decisions_total": ( + "Terminal outcome of each executed CLI lane route decision." + ), + "hermes_cli_provider_fallbacks_total": ( + "Cross-provider failovers taken by the CLI lane, by failure reason." + ), + "hermes_cli_router_selection_failures_total": ( + "Switchyard route selections that failed and blocked a card as transient." + ), + "hermes_cli_quota_fetch_failures_total": ( + "Quota snapshot reads that failed; routing then fails open." + ), + "hermes_cli_quota_remaining_percent": ( + "Binding remaining-quota percent per provider as seen by the CLI lane." + ), + "hermes_cli_quota_reset_timestamp_seconds": ( + "Unix reset time of the binding quota window as seen by the CLI lane." + ), + "hermes_cli_provider_soft_excluded": ( + "Whether new automatic work is currently routed away from a provider." + ), +} +COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total")) + + +def _escape_label(value: str) -> str: + """Escape one Prometheus label value.""" + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +class LaneMetricsRegistry: + """Thread-safe counter and gauge store rendered in Prometheus text format.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {} + + def _key( + self, name: str, labels: dict[str, str] | None + ) -> tuple[str, tuple[tuple[str, str], ...]]: + return name, tuple(sorted((labels or {}).items())) + + def increment( + self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0 + ) -> None: + key = self._key(name, labels) + with self._lock: + self._values[key] = self._values.get(key, 0.0) + amount + + def set_value( + self, name: str, labels: dict[str, str] | None, value: float + ) -> None: + with self._lock: + self._values[self._key(name, labels)] = value + + def render(self) -> bytes: + """Render every recorded sample with stable ordering.""" + with self._lock: + values = dict(self._values) + lines: list[str] = [] + for name in sorted({name for name, _ in values}): + kind = "counter" if name in COUNTER_NAMES else "gauge" + lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}")) + for (sample_name, labels), value in sorted(values.items()): + if sample_name != name: + continue + rendered = ",".join( + f'{key}="{_escape_label(text)}"' for key, text in labels + ) + body = f"{{{rendered}}}" if rendered else "" + lines.append(f"{name}{body} {value:.12g}") + return ("\n".join(lines) + "\n").encode("utf-8") + + +METRICS = LaneMetricsRegistry() + + +def record_route_decision( + provider: str, effort: str, classifier: str, outcome: str +) -> None: + """Count one executed route with its terminal outcome.""" + METRICS.increment( + "hermes_cli_route_decisions_total", + { + "provider": provider, + "effort": effort, + "classifier": classifier, + "outcome": outcome, + }, + ) + + +def record_provider_fallback( + from_provider: str, to_provider: str, reason: str +) -> None: + """Count one cross-provider failover with its classified reason.""" + METRICS.increment( + "hermes_cli_provider_fallbacks_total", + { + "from_provider": from_provider, + "to_provider": to_provider, + "reason": reason, + }, + ) + + +def record_router_selection_failure() -> None: + """Count one Switchyard outage observed at a selection boundary.""" + METRICS.increment("hermes_cli_router_selection_failures_total") + + +def record_quota_fetch_failure() -> None: + """Count one unavailable quota snapshot; routing continues fail-open.""" + METRICS.increment("hermes_cli_quota_fetch_failures_total") + + +def record_provider_quota( + provider: str, remaining_percent: float, reset_timestamp: float | None +) -> None: + """Publish the binding quota window the lane based its routing on.""" + labels = {"provider": provider} + METRICS.set_value( + "hermes_cli_quota_remaining_percent", labels, remaining_percent + ) + if reset_timestamp is not None: + METRICS.set_value( + "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp + ) + + +def record_soft_exclusion(provider: str, excluded: bool) -> None: + """Publish whether new automatic work avoids this provider right now.""" + METRICS.set_value( + "hermes_cli_provider_soft_excluded", + {"provider": provider}, + 1.0 if excluded else 0.0, + ) + + +def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None: + """Serve lane metrics on the pod-local scrape port without blocking dispatch.""" + selected_port = ( + int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011")) + if port is None + else port + ) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + if self.path == "/metrics": + payload = METRICS.render() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + elif self.path == "/healthz": + self.send_response(200) + self.end_headers() + else: + self.send_error(404) + + def log_message(self, _format: str, *_args: object) -> None: + return + + try: + server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler) + except OSError as error: + print( + f"cli lane metrics server unavailable: {error}", + file=sys.stderr, + flush=True, + ) + return None + threading.Thread(target=server.serve_forever, daemon=True).start() + return server diff --git a/services/hermes/scripts/cli_lane_provider.py b/services/hermes/scripts/cli_lane_provider.py index 07a3d02b..c7d57851 100644 --- a/services/hermes/scripts/cli_lane_provider.py +++ b/services/hermes/scripts/cli_lane_provider.py @@ -197,6 +197,10 @@ def _terminate_worker_process( def _base_env() -> dict[str, str]: """Build a worker environment while preserving Vault-backed CLI homes.""" env = os.environ.copy() + # Subscription CLIs only: a vendor API key in the environment could + # silently flip a worker onto a metered path, so drop every key here. + for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"): + env.pop(secret, None) env.update( { "HOME": str(DATA_ROOT / "home"), diff --git a/services/hermes/scripts/cli_lane_quota.py b/services/hermes/scripts/cli_lane_quota.py new file mode 100644 index 00000000..4312d052 --- /dev/null +++ b/services/hermes/scripts/cli_lane_quota.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Quota-aware soft routing for automatic CLI lane provider selection.""" + +from __future__ import annotations + +import re +import sys +import urllib.request +from dataclasses import dataclass +from typing import Callable + +from cli_lane_config import ( + QUOTA_METRICS_URL, + QUOTA_MIN_REMAINING_PERCENT_DEFAULT, + kanban_setting, +) +from cli_lane_metrics import ( + record_provider_quota, + record_quota_fetch_failure, + record_soft_exclusion, +) +from cli_lane_routing import fresh_unavailable_provider + + +# Exporter accounts are labeled by vendor; the lane routes by CLI provider. +QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"} +# Gate only on account-wide windows: a model-specific Claude window (for +# example seven_day_opus) must not veto the whole provider. +GATED_QUOTA_WINDOWS = { + "codex": None, + "claude": ("five_hour", "seven_day"), +} +_SAMPLE_PATTERN = re.compile( + r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)" + r"\{([^}]*)\} (-?[0-9.eE+]+)$" +) +_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"') + + +@dataclass(frozen=True) +class ProviderQuota: + """One provider's binding remaining quota as seen by the lane.""" + + remaining_percent: float + reset_timestamp: float | None + + +@dataclass(frozen=True) +class SelectionConstraint: + """One provider exclusion decided at a routing boundary, with evidence.""" + + exclude_provider: str | None + exclude_reason: str | None + source: str | None + notes: tuple[str, ...] + + +def _gated_window(provider: str, labels: dict[str, str]) -> str | None: + """Return the window key when a sample belongs to a gated account limit.""" + if labels.get("limit") != provider: + return None + window = labels.get("window", "") + allowed = GATED_QUOTA_WINDOWS[provider] + if allowed is not None and window not in allowed: + return None + return window + + +def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]: + """Reduce exporter samples to each provider's tightest account window.""" + remaining: dict[tuple[str, str], float] = {} + resets: dict[tuple[str, str], float] = {} + fetch_success: dict[str, bool] = {} + for line in text.splitlines(): + match = _SAMPLE_PATTERN.match(line.strip()) + if not match: + continue + metric, raw_labels, raw_value = match.groups() + labels = dict(_LABEL_PATTERN.findall(raw_labels)) + provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", "")) + if provider is None: + continue + value = float(raw_value) + if metric == "fetch_success": + fetch_success[provider] = value >= 1.0 + continue + window = _gated_window(provider, labels) + if window is None: + continue + if metric == "remaining_percent": + remaining[(provider, window)] = value + else: + resets[(provider, window)] = value + snapshot: dict[str, ProviderQuota] = {} + for provider in QUOTA_PROVIDER_LABELS.values(): + windows = { + window: value + for (candidate, window), value in remaining.items() + if candidate == provider + } + if not windows or not fetch_success.get(provider, True): + # Failed or stale exporter fetches leave this provider ungated. + continue + binding = min(windows, key=lambda window: (windows[window], window)) + snapshot[provider] = ProviderQuota( + remaining_percent=windows[binding], + reset_timestamp=resets.get((provider, binding)), + ) + return snapshot + + +def fetch_quota_snapshot( + url: str = QUOTA_METRICS_URL, + open_url: Callable = urllib.request.urlopen, +) -> dict[str, ProviderQuota]: + """Read the in-pod usage exporter; empty means the signal is unavailable.""" + try: + with open_url(url, timeout=10) as response: + text = response.read(1 << 20).decode("utf-8", errors="replace") + snapshot = parse_quota_metrics(text) + except (OSError, ValueError) as error: + record_quota_fetch_failure() + print( + f"quota snapshot unavailable; routing fails open: {type(error).__name__}", + file=sys.stderr, + flush=True, + ) + return {} + if not snapshot: + record_quota_fetch_failure() + print( + "quota snapshot carried no gated windows; routing fails open", + file=sys.stderr, + flush=True, + ) + return snapshot + + +def quota_soft_exclusion( + snapshot: dict[str, ProviderQuota], threshold: float +) -> tuple[str | None, str | None]: + """Choose at most one provider to soft-exclude from new automatic work.""" + below = { + provider: quota.remaining_percent + for provider, quota in snapshot.items() + if quota.remaining_percent < threshold + } + if not below: + return None, None + excluded = min(below, key=lambda provider: (below[provider], provider)) + if len(below) == len(snapshot) and len(snapshot) > 1: + preferred = next(name for name in snapshot if name != excluded) + note = ( + "Quota degraded: every provider is below the " + f"{threshold:.6g}% remaining threshold; preferring {preferred} " + f"({snapshot[preferred].remaining_percent:.6g}% left) over " + f"{excluded} ({below[excluded]:.6g}% left) for new auto work." + ) + return excluded, note + note = ( + f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining " + f"(threshold {threshold:.6g}%); routing new auto work to the other " + "provider while active work finishes." + ) + return excluded, note + + +def selection_constraint( + assignee: str, + *, + snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None, + now: float | None = None, +) -> SelectionConstraint: + """Combine provider health and quota gating for one selection boundary.""" + if assignee != "cli-auto": + # Explicit lanes are operator decisions; they are never re-routed. + return SelectionConstraint(None, None, None, ()) + snapshot = (snapshot_from or fetch_quota_snapshot)() + resets: dict[str, float] = {} + for provider, quota in snapshot.items(): + record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp) + if quota.reset_timestamp is not None: + resets[provider] = quota.reset_timestamp + health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets) + if health_excluded is not None: + return SelectionConstraint( + health_excluded, + "is unavailable according to fresh native health", + "health", + (), + ) + threshold = kanban_setting( + "provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT + ) + excluded, note = quota_soft_exclusion(snapshot, threshold) + for provider in snapshot: + record_soft_exclusion(provider, provider == excluded) + if excluded is None or note is None: + return SelectionConstraint(None, None, None, ()) + return SelectionConstraint( + excluded, + "is below its remaining-quota routing threshold", + "quota", + (note,), + ) diff --git a/services/hermes/scripts/cli_lane_routing.py b/services/hermes/scripts/cli_lane_routing.py index 3ee58d34..63819981 100644 --- a/services/hermes/scripts/cli_lane_routing.py +++ b/services/hermes/scripts/cli_lane_routing.py @@ -31,12 +31,49 @@ def parse_assignee(assignee: str) -> tuple[str | None, str | None]: raise ValueError(f"unsupported external lane assignee: {assignee}") return match.group(1), match.group(2) -def fresh_unavailable_provider(now: float | None = None) -> str | None: +def _health_number(value: Any) -> float | None: + """Return a plain numeric health field, rejecting booleans and strings.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + +def _quota_reset_readmits( + health: dict[str, Any], current: float, reset_timestamp: float | None +) -> bool: + """A passed provider quota reset ends a capacity cooldown early.""" + if health.get("authenticated") is False: + # An authentication failure is not repaired by a quota window reset. + return False + failed_at = _health_number(health.get("failed_at")) + return ( + reset_timestamp is not None + and failed_at is not None + and failed_at < reset_timestamp <= current + ) + +def fresh_unavailable_provider( + now: float | None = None, + quota_resets: dict[str, float] | None = None, +) -> str | None: """Return one recently proven-down provider for automatic route exclusion.""" current = time.time() if now is None else now + resets = quota_resets or {} unavailable: list[str] = [] for provider, path in PROVIDER_HEALTH_PATHS.items(): health = load_json(path) + if health.get("state") not in ("unavailable", "capacity-limited"): + continue + cooldown_until = _health_number(health.get("cooldown_until")) + if cooldown_until is not None: + # Lane-recorded failures re-enter only after the full cooldown has + # elapsed (hysteresis), a quota reset passed, or fresh success + # rewrote the state above. + if current >= cooldown_until or _quota_reset_readmits( + health, current, resets.get(provider) + ): + continue + unavailable.append(provider) + continue try: age = current - path.stat().st_mtime except OSError: @@ -46,10 +83,7 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None: if health.get("authenticated") is False else PROVIDER_HEALTH_MAX_AGE_SECONDS ) - if ( - 0 <= age <= max_age - and health.get("state") == "unavailable" - ): + if 0 <= age <= max_age: unavailable.append(provider) # If both providers are down, keep the normal attempt/fallback path so the # card records authoritative current errors instead of trusting snapshots. diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 84aa185d..cc466cb0 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -32,6 +32,11 @@ from cli_lane_board import ( ) from cli_lane_config import ( ARTIFACT_GC_INTERVAL_SECONDS, + AUTH_COOLDOWN_SECONDS_DEFAULT, + CAPACITY_COOLDOWN_SECONDS_DEFAULT, + QUOTA_METRICS_URL, + QUOTA_MIN_REMAINING_PERCENT_DEFAULT, + kanban_setting, ARTIFACT_RETENTION_AGE_SECONDS, ARTIFACT_RETENTION_BYTES, ARTIFACT_RETENTION_COUNT, @@ -67,6 +72,35 @@ from cli_lane_evidence import ( _write_json_noreplace, ) from cli_lane_execution import execute_claim +from cli_lane_failover import ( + FailoverOutcome, + _block_transient, + _routed_or_blocked, + capacity_failover, +) +from cli_lane_health import ( + classify_capacity_failure, + record_provider_failure, + record_provider_success, +) +from cli_lane_metrics import ( + METRICS, + record_provider_fallback, + record_provider_quota, + record_quota_fetch_failure, + record_route_decision, + record_router_selection_failure, + record_soft_exclusion, + start_metrics_server, +) +from cli_lane_quota import ( + ProviderQuota, + SelectionConstraint, + fetch_quota_snapshot, + parse_quota_metrics, + quota_soft_exclusion, + selection_constraint, +) from cli_lane_files import ( _candidate_path, _fsync_directory, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index cfcc0e43..78a4e13d 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -26,12 +26,16 @@ "services/hermes/scripts/cli_lane_dispatch.py", "services/hermes/scripts/cli_lane_evidence.py", "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_failover.py", "services/hermes/scripts/cli_lane_files.py", "services/hermes/scripts/cli_lane_finalization.py", "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_health.py", + "services/hermes/scripts/cli_lane_metrics.py", "services/hermes/scripts/cli_lane_prompt.py", "services/hermes/scripts/cli_lane_provider.py", "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_quota.py", "services/hermes/scripts/cli_lane_records.py", "services/hermes/scripts/cli_lane_recovery.py", "services/hermes/scripts/cli_lane_retention.py", @@ -45,10 +49,10 @@ "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", - "testing/tests/test_publish_test_metrics.py", - "testing/tests/test_supply_chain_report.py", - "testing/tests/test_semgrep_report.py", - "testing/tests/test_quality_contract.py", + "testing/tests/test_publish_test_metrics.py", + "testing/tests/test_supply_chain_report.py", + "testing/tests/test_semgrep_report.py", + "testing/tests/test_quality_contract.py", "testing/tests/test_quality_gate.py" ], "lint_paths": [ @@ -66,12 +70,16 @@ "services/hermes/scripts/cli_lane_dispatch.py", "services/hermes/scripts/cli_lane_evidence.py", "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_failover.py", "services/hermes/scripts/cli_lane_files.py", "services/hermes/scripts/cli_lane_finalization.py", "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_health.py", + "services/hermes/scripts/cli_lane_metrics.py", "services/hermes/scripts/cli_lane_prompt.py", "services/hermes/scripts/cli_lane_provider.py", "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_quota.py", "services/hermes/scripts/cli_lane_records.py", "services/hermes/scripts/cli_lane_recovery.py", "services/hermes/scripts/cli_lane_retention.py", @@ -216,12 +224,16 @@ "services/hermes/scripts/cli_lane_dispatch.py", "services/hermes/scripts/cli_lane_evidence.py", "services/hermes/scripts/cli_lane_execution.py", + "services/hermes/scripts/cli_lane_failover.py", "services/hermes/scripts/cli_lane_files.py", "services/hermes/scripts/cli_lane_finalization.py", "services/hermes/scripts/cli_lane_goal.py", + "services/hermes/scripts/cli_lane_health.py", + "services/hermes/scripts/cli_lane_metrics.py", "services/hermes/scripts/cli_lane_prompt.py", "services/hermes/scripts/cli_lane_provider.py", "services/hermes/scripts/cli_lane_quarantine.py", + "services/hermes/scripts/cli_lane_quota.py", "services/hermes/scripts/cli_lane_records.py", "services/hermes/scripts/cli_lane_recovery.py", "services/hermes/scripts/cli_lane_retention.py", From 6df688adcc212988a281e5bb8858a0531c638971 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:31:12 -0300 Subject: [PATCH 37/46] hermes: test quota gating, cooldown recovery, and failover policy Deterministic coverage for the quota-aware lane: threshold boundaries (14.9/15/15.1), both-below preference, fetch-failure fail-open, cooldown elapsed-vs-not hysteresis, quota-reset recovery (never for auth), explicit fail-closed in both directions, bounded double-failure block, failure-reason classification, metrics emission, and worker env key stripping. Based on PR #15 (fix/hermes-result-decomposition-reliability). Co-Authored-By: Claude Opus 4.8 --- testing/tests/test_hermes_agent_security.py | 5 +- .../tests/test_hermes_cli_dispatch_runtime.py | 4 + .../tests/test_hermes_cli_execution_edges.py | 6 +- testing/tests/test_hermes_cli_fallback.py | 2 +- .../tests/test_hermes_cli_goal_execution.py | 2 +- testing/tests/test_hermes_cli_lane_metrics.py | 116 +++++ .../tests/test_hermes_cli_provider_health.py | 200 ++++++++ .../tests/test_hermes_cli_quota_failover.py | 440 ++++++++++++++++++ testing/tests/test_hermes_cli_quota_gating.py | 299 ++++++++++++ 9 files changed, 1068 insertions(+), 6 deletions(-) create mode 100644 testing/tests/test_hermes_cli_lane_metrics.py create mode 100644 testing/tests/test_hermes_cli_provider_health.py create mode 100644 testing/tests/test_hermes_cli_quota_failover.py create mode 100644 testing/tests/test_hermes_cli_quota_gating.py diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py index cf23a2d0..1d713203 100644 --- a/testing/tests/test_hermes_agent_security.py +++ b/testing/tests/test_hermes_agent_security.py @@ -179,7 +179,10 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces() "podSelector": {"matchLabels": {"app": "server"}}, } ], - "ports": [{"protocol": "TCP", "port": 9010}], + "ports": [ + {"protocol": "TCP", "port": 9010}, + {"protocol": "TCP", "port": 9011}, + ], }, ] assert isolation["spec"]["egress"] == [{}] diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py index 06873f0a..3ffc0b39 100644 --- a/testing/tests/test_hermes_cli_dispatch_runtime.py +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -161,6 +161,7 @@ def test_ready_dispatch_loop_submits_and_reaps_failed_workers( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None) ready = lanes.KanbanCapabilities(True, True, True) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) @@ -196,6 +197,7 @@ def test_deferred_dispatch_health_never_claims_new_work( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None) deferred = lanes.KanbanCapabilities(True, False, False) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred) @@ -251,6 +253,7 @@ def test_two_loop_api_transition_refreshes_dispatch_and_health( monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") recoveries = [] monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover")) + monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) claim_states = [] @@ -369,6 +372,7 @@ def test_dispatch_loop_survives_board_registry_scan_failure( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) + monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None) ready = lanes.KanbanCapabilities(True, True, True) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py index 7873c86c..d1ffbda8 100644 --- a/testing/tests/test_hermes_cli_execution_edges.py +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -58,7 +58,7 @@ def test_transient_callback_storage_errors_do_not_kill_provider( monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: "claude") + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: "claude") route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) @@ -246,7 +246,7 @@ def test_capacity_fallback_without_evidence_blocks_with_provider_output( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None) codex = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) claude = lanes.Route("claude", "fable", "high", "p", "c", "r", 1, ()) monkeypatch.setattr( @@ -298,7 +298,7 @@ def test_worker_blockers_become_the_block_reason_without_goal_mode( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None) route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) structured = { diff --git a/testing/tests/test_hermes_cli_fallback.py b/testing/tests/test_hermes_cli_fallback.py index e2a8141e..e8e4bc1d 100644 --- a/testing/tests/test_hermes_cli_fallback.py +++ b/testing/tests/test_hermes_cli_fallback.py @@ -73,7 +73,7 @@ def test_goal_card_continues_after_local_judge_rejects_progress( return codex_xhigh monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None) reports = [ lanes.ProcessResult(1, "authentication expired", None, True), lanes.ProcessResult( diff --git a/testing/tests/test_hermes_cli_goal_execution.py b/testing/tests/test_hermes_cli_goal_execution.py index 749a3d12..b8f1b1a5 100644 --- a/testing/tests/test_hermes_cli_goal_execution.py +++ b/testing/tests/test_hermes_cli_goal_execution.py @@ -191,7 +191,7 @@ def test_goal_loop_recovers_from_corrupt_rejection_history( monkeypatch.setattr( lanes, "fresh_unavailable_provider", - lambda: probes.append("health") and None, + lambda *_args, **_kwargs: probes.append("health") and None, ) route = lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () diff --git a/testing/tests/test_hermes_cli_lane_metrics.py b/testing/tests/test_hermes_cli_lane_metrics.py new file mode 100644 index 00000000..ff832065 --- /dev/null +++ b/testing/tests/test_hermes_cli_lane_metrics.py @@ -0,0 +1,116 @@ +"""Lane metrics registry semantics and the pod-local scrape endpoint.""" + +from __future__ import annotations + +import urllib.error +import urllib.request + +from testing.tests.test_hermes_cli_support import lanes + + +def test_registry_renders_counters_and_gauges_with_stable_labels(): + registry = lanes.LaneMetricsRegistry() + registry.increment( + "hermes_cli_route_decisions_total", + {"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"}, + ) + registry.increment( + "hermes_cli_route_decisions_total", + {"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"}, + ) + registry.increment("hermes_cli_quota_fetch_failures_total") + registry.set_value( + "hermes_cli_quota_remaining_percent", {"provider": 'we"ird\n\\'}, 12.5 + ) + rendered = registry.render().decode("utf-8") + + assert "# TYPE hermes_cli_route_decisions_total counter" in rendered + assert "# TYPE hermes_cli_quota_remaining_percent gauge" in rendered + assert ( + 'hermes_cli_route_decisions_total{classifier="c",effort="high",' + 'outcome="completed",provider="codex"} 2' in rendered + ) + assert "hermes_cli_quota_fetch_failures_total 1" in rendered + assert ( + 'hermes_cli_quota_remaining_percent{provider="we\\"ird\\n\\\\"} 12.5' + in rendered + ) + + +def test_recording_helpers_feed_the_shared_registry(): + lanes.record_route_decision("codex", "xhigh", "metrics-test", "completed") + lanes.record_provider_fallback("codex", "claude", "rate-limit") + lanes.record_router_selection_failure() + lanes.record_quota_fetch_failure() + lanes.record_provider_quota("metrics-test-provider", 42.0, 777.0) + lanes.record_provider_quota("metrics-test-no-reset", 41.0, None) + lanes.record_soft_exclusion("metrics-test-provider", True) + rendered = lanes.METRICS.render().decode("utf-8") + + assert ( + 'hermes_cli_route_decisions_total{classifier="metrics-test",' + 'effort="xhigh",outcome="completed",provider="codex"} 1' in rendered + ) + assert ( + 'hermes_cli_provider_fallbacks_total{from_provider="codex",' + 'reason="rate-limit",to_provider="claude"}' in rendered + ) + assert "hermes_cli_router_selection_failures_total" in rendered + assert "hermes_cli_quota_fetch_failures_total" in rendered + assert ( + 'hermes_cli_quota_remaining_percent{provider="metrics-test-provider"} 42' + in rendered + ) + assert ( + 'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-provider"} 777' + in rendered + ) + assert ( + 'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-no-reset"}' + not in rendered + ) + assert ( + 'hermes_cli_provider_soft_excluded{provider="metrics-test-provider"} 1' + in rendered + ) + + +def test_metrics_server_serves_scrapes_health_and_rejects_unknown_paths(): + server = lanes.start_metrics_server(0) + assert server is not None + try: + port = server.server_address[1] + lanes.record_route_decision("codex", "low", "scrape-test", "completed") + with urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics") as response: + assert response.status == 200 + assert b'classifier="scrape-test"' in response.read() + with urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz") as response: + assert response.status == 200 + try: + urllib.request.urlopen(f"http://127.0.0.1:{port}/nope") + raise AssertionError("unknown paths must 404") + except urllib.error.HTTPError as error: + assert error.code == 404 + finally: + server.shutdown() + server.server_close() + + +def test_metrics_server_reads_port_from_environment(monkeypatch): + monkeypatch.setenv("HERMES_CLI_LANE_METRICS_PORT", "0") + server = lanes.start_metrics_server() + assert server is not None + server.shutdown() + server.server_close() + + +def test_metrics_server_declines_conflicting_port(capsys): + first = lanes.start_metrics_server(0) + assert first is not None + try: + second = lanes.start_metrics_server(first.server_address[1]) + assert second is None + assert "metrics server unavailable" in capsys.readouterr().err + finally: + first.shutdown() + first.server_close() diff --git a/testing/tests/test_hermes_cli_provider_health.py b/testing/tests/test_hermes_cli_provider_health.py new file mode 100644 index 00000000..01bb810e --- /dev/null +++ b/testing/tests/test_hermes_cli_provider_health.py @@ -0,0 +1,200 @@ +"""Lane-side provider health writes, cooldown hysteresis, and recovery.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + json, + lanes, + pytest, +) + + +@pytest.fixture +def health_paths(tmp_path: Path, monkeypatch) -> dict[str, Path]: + paths = { + "codex": tmp_path / "health/codex.json", + "claude": tmp_path / "health/claude.json", + } + monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) + monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) + return paths + + +@pytest.mark.parametrize( + ("output", "expected"), + [ + ("You have hit your usage limit for this billing cycle.", "quota"), + ("credit balance exhausted", "quota"), + ("HTTP 429: too many requests", "rate-limit"), + ("overloaded_error (529)", "rate-limit"), + ("OAuth token expired; run codex login", "auth"), + ("401 Unauthorized", "auth"), + ("stream disconnected before completion", "transport"), + ], +) +def test_capacity_failures_are_classified_actionably(output: str, expected: str): + assert lanes.classify_capacity_failure(output) == expected + + +def test_capacity_failure_records_cooldown_with_defaults(health_paths): + until = lanes.record_provider_failure("codex", "quota", now=1000.0) + + value = json.loads(health_paths["codex"].read_text()) + assert until == 1300.0 + assert value["state"] == "capacity-limited" + assert value["authenticated"] is True + assert value["failure_reason"] == "quota" + assert value["failed_at"] == 1000.0 + assert value["cooldown_until"] == 1300.0 + assert value["source"] == "cli-lane-runner" + + +def test_auth_failure_uses_longer_cooldown_and_marks_unauthenticated(health_paths): + until = lanes.record_provider_failure("claude", "auth", now=1000.0) + + value = json.loads(health_paths["claude"].read_text()) + assert until == 4600.0 + assert value["state"] == "unavailable" + assert value["authenticated"] is False + + +def test_cooldowns_are_configurable_through_the_kanban_block( + health_paths, tmp_path: Path +): + (tmp_path / "config.yaml").write_text( + "kanban:\n" + " provider_capacity_cooldown_seconds: 60\n" + " provider_auth_cooldown_seconds: 120\n" + ) + + assert lanes.record_provider_failure("codex", "rate-limit", now=1000.0) == 1060.0 + assert lanes.record_provider_failure("codex", "auth", now=1000.0) == 1120.0 + + +def test_lane_writes_merge_with_broker_health_snapshots(health_paths): + health_paths["codex"].parent.mkdir(parents=True) + health_paths["codex"].write_text( + json.dumps({"transport": "codex-chatgpt-subscription", "state": "available"}) + ) + + lanes.record_provider_failure("codex", "quota", now=1000.0) + + value = json.loads(health_paths["codex"].read_text()) + assert value["transport"] == "codex-chatgpt-subscription" + assert value["state"] == "capacity-limited" + + +def test_success_clears_failure_evidence_and_readmits(health_paths): + lanes.record_provider_failure("codex", "quota", now=1000.0) + lanes.record_provider_success("codex") + + value = json.loads(health_paths["codex"].read_text()) + assert value["state"] == "available" + assert value["authenticated"] is True + for stale in ("failure_reason", "failed_at", "cooldown_until"): + assert stale not in value + assert lanes.fresh_unavailable_provider(now=1001.0) is None + + +def test_health_write_failures_never_break_the_lane(health_paths, tmp_path: Path): + blocker = tmp_path / "health" + blocker.write_text("a file where the directory must go") + + lanes.record_provider_failure("codex", "quota", now=1000.0) + + +def test_broker_reported_capacity_limited_state_is_excluded(health_paths): + health_paths["codex"].parent.mkdir(parents=True) + health_paths["codex"].write_text('{"state":"capacity-limited"}\n') + + now = health_paths["codex"].stat().st_mtime + assert lanes.fresh_unavailable_provider(now=now) == "codex" + + +def test_cooldown_must_fully_elapse_before_readmission(health_paths): + lanes.record_provider_failure("codex", "quota", now=1000.0) + + assert lanes.fresh_unavailable_provider(now=1299.9) == "codex" + assert lanes.fresh_unavailable_provider(now=1300.0) is None + + +def test_quota_reset_readmits_capacity_limited_provider_early(health_paths): + lanes.record_provider_failure("codex", "quota", now=1000.0) + + # Reset before the failure proves nothing; a future reset has not passed. + assert ( + lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 900.0}) + == "codex" + ) + assert ( + lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 1150.0}) + == "codex" + ) + # A reset between the failure and now means the window rolled over. + assert ( + lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0}) + is None + ) + + +def test_quota_reset_never_readmits_auth_failures(health_paths): + lanes.record_provider_failure("codex", "auth", now=1000.0) + + assert ( + lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0}) + == "codex" + ) + + +def test_both_providers_in_cooldown_defer_to_live_attempts(health_paths): + lanes.record_provider_failure("codex", "quota", now=1000.0) + lanes.record_provider_failure("claude", "quota", now=1000.0) + + assert lanes.fresh_unavailable_provider(now=1100.0) is None + + +def test_vanished_health_file_is_not_excluded(health_paths, monkeypatch): + monkeypatch.setattr(lanes, "load_json", lambda _path: {"state": "unavailable"}) + + assert lanes.fresh_unavailable_provider(now=1000.0) is None + + +def test_malformed_cooldown_falls_back_to_freshness_window(health_paths): + health_paths["claude"].parent.mkdir(parents=True) + health_paths["claude"].write_text( + '{"state":"capacity-limited","cooldown_until":true}\n' + ) + + now = health_paths["claude"].stat().st_mtime + assert lanes.fresh_unavailable_provider(now=now) == "claude" + assert lanes.fresh_unavailable_provider(now=now + 10 * 60) is None + + +def test_health_number_rejects_booleans_and_strings(): + assert lanes._health_number(True) is None + assert lanes._health_number("5") is None + assert lanes._health_number(5) == 5.0 + + +def test_worker_environment_strips_metered_api_keys(monkeypatch): + for name in ( + "ANTHROPIC_API_KEY", + "CLAUDE_API_KEY", + "OPENAI_API_KEY", + "API_SERVER_KEY", + ): + monkeypatch.setenv(name, "must-not-leak") + monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex") + + env = lanes._base_env() + + for name in ( + "ANTHROPIC_API_KEY", + "CLAUDE_API_KEY", + "OPENAI_API_KEY", + "API_SERVER_KEY", + ): + assert name not in env + assert env["CODEX_HOME"] == "/runtime-access/codex" + assert env["GIT_TERMINAL_PROMPT"] == "0" diff --git a/testing/tests/test_hermes_cli_quota_failover.py b/testing/tests/test_hermes_cli_quota_failover.py new file mode 100644 index 00000000..3cc54850 --- /dev/null +++ b/testing/tests/test_hermes_cli_quota_failover.py @@ -0,0 +1,440 @@ +"""Fail-closed manual lanes, bounded failover, and router-outage blocking.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + json, + lanes, + nullcontext, + sys, +) + + +COMPLETED_RESULT = { + "status": "completed", + "summary": "Work finished with evidence.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +class _Connection: + def close(self): + return None + + +def _lane_board(tmp_path: Path, task, comments: list, calls: list) -> SimpleNamespace: + return SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: _Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, f"wt/{task.id}"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Finish the assigned card.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: ( + calls.append(("complete", kwargs)) or True + ), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + + +def _isolate_lane(tmp_path: Path, monkeypatch, board) -> dict[str, Path]: + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board)) + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) + health_paths = { + "codex": tmp_path / "provider-health/codex.json", + "claude": tmp_path / "provider-health/claude.json", + } + monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", health_paths) + monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {}) + return health_paths + + +def _route(provider: str, effort: str): + model = "gpt-5.6-sol" if provider == "codex" else "claude-opus-5" + return lanes.Route( + provider, model, effort, f"{provider}-{effort}", "switchyard-manual", "r", 1, () + ) + + +def _run_manual_fail_closed(tmp_path, monkeypatch, assignee, provider, output): + task = SimpleNamespace( + id="t_manual", + status="running", + result=None, + current_run_id=31, + assignee=assignee, + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + health_paths = _isolate_lane(tmp_path, monkeypatch, board) + route_calls: list = [] + monkeypatch.setattr( + lanes, + "select_route", + lambda _prompt, lane, **kwargs: ( + route_calls.append((lane, kwargs)) or _route(provider, "high") + ), + ) + runs: list = [] + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: ( + runs.append(provider) or lanes.ProcessResult(1, output, None, True) + ), + ) + + lanes.execute_claim("cassandra", "t_manual") + return calls, comments, route_calls, runs, health_paths + + +def test_manual_codex_assignee_fails_closed_without_provider_switch( + tmp_path: Path, monkeypatch +): + calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( + tmp_path, monkeypatch, "cli-codex-high", "codex", "HTTP 429 rate limit hit" + ) + + assert runs == ["codex"] + assert [lane for lane, _ in route_calls] == ["cli-codex-high"] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Manually pinned provider codex" in kwargs["reason"] + assert "cli-codex-high" in kwargs["reason"] + assert "rate-limit" in kwargs["reason"] + health = json.loads(health_paths["codex"].read_text()) + assert health["state"] == "capacity-limited" + assert not health_paths["claude"].exists() + + +def test_manual_claude_assignee_fails_closed_without_provider_switch( + tmp_path: Path, monkeypatch +): + calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( + tmp_path, monkeypatch, "cli-claude-low", "claude", "usage limit exhausted" + ) + + assert runs == ["claude"] + assert [lane for lane, _ in route_calls] == ["cli-claude-low"] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Manually pinned provider claude" in kwargs["reason"] + assert "quota" in kwargs["reason"] + health = json.loads(health_paths["claude"].read_text()) + assert health["state"] == "capacity-limited" + assert not health_paths["codex"].exists() + + +def test_auto_failover_preserves_effort_and_records_cooldown( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_auto", + status="running", + result=None, + current_run_id=32, + assignee="cli-auto", + max_runtime_seconds=120, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + health_paths = _isolate_lane(tmp_path, monkeypatch, board) + route_calls: list = [] + + def select_route(_prompt, lane, **kwargs): + route_calls.append((lane, kwargs)) + if lane == "cli-auto": + return lanes.Route( + "codex", "gpt-5.6-terra", "xhigh", "codex-xhigh", + "switchyard-classifier", "vote", 1, (), + ) + assert lane == "cli-claude-xhigh" + return _route("claude", "xhigh") + + monkeypatch.setattr(lanes, "select_route", select_route) + fallbacks: list = [] + monkeypatch.setattr( + lanes, + "record_provider_fallback", + lambda source, target, reason: fallbacks.append((source, target, reason)), + ) + reports = [ + lanes.ProcessResult(1, "You have hit your usage limit.", None, True), + lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), + ] + monkeypatch.setattr( + lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0) + ) + + lanes.execute_claim("cassandra", "t_auto") + + assert reports == [] + assert calls and calls[0][0] == "complete" + assert fallbacks == [("codex", "claude", "quota")] + assert any( + "Provider fallback: codex -> claude after a quota failure" in item + for item in comments + ) + codex_health = json.loads(health_paths["codex"].read_text()) + assert codex_health["state"] == "capacity-limited" + assert codex_health["cooldown_until"] == codex_health["failed_at"] + 300.0 + claude_health = json.loads(health_paths["claude"].read_text()) + assert claude_health["state"] == "available" + + +def test_double_capacity_failure_blocks_transient_with_both_reasons( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_double", + status="running", + result=None, + current_run_id=33, + assignee="cli-auto", + max_runtime_seconds=120, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + health_paths = _isolate_lane(tmp_path, monkeypatch, board) + + def select_route(_prompt, lane, **_kwargs): + return ( + _route("codex", "high") if lane == "cli-auto" else _route("claude", "high") + ) + + monkeypatch.setattr(lanes, "select_route", select_route) + decisions: list = [] + monkeypatch.setattr( + lanes, + "record_route_decision", + lambda provider, effort, classifier, outcome: decisions.append( + (provider, effort, outcome) + ), + ) + reports = [ + lanes.ProcessResult(1, "usage limit reached for this window", None, True), + lanes.ProcessResult(1, "invalid oauth token, authentication failed", None, True), + ] + runs: list = [] + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: runs.append(1) or reports.pop(0), + ) + + lanes.execute_claim("cassandra", "t_double") + + assert len(runs) == 2 + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Both hosted providers failed" in kwargs["reason"] + assert "codex: quota" in kwargs["reason"] + assert "claude: auth" in kwargs["reason"] + assert "never falls to a metered or local path" in kwargs["reason"] + assert decisions == [ + ("codex", "high", "capacity-failure"), + ("claude", "high", "capacity-failure"), + ] + assert json.loads(health_paths["codex"].read_text())["state"] == "capacity-limited" + claude_health = json.loads(health_paths["claude"].read_text()) + assert claude_health["state"] == "unavailable" + assert claude_health["authenticated"] is False + + +def test_router_outage_blocks_transient_before_any_provider_run( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_router", + status="running", + result=None, + current_run_id=34, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("Switchyard worker routing failed: refused") + ), + ) + outages: list = [] + monkeypatch.setattr( + lanes, "record_router_selection_failure", lambda: outages.append(1) + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("no provider may run without a route") + ), + ) + + lanes.execute_claim("cassandra", "t_router") + + assert outages == [1] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] + + +def test_router_outage_during_fallback_selection_blocks_transient( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_router_fb", + status="running", + result=None, + current_run_id=35, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + selections: list = [] + + def select_route(_prompt, lane, **_kwargs): + selections.append(lane) + if lane == "cli-auto": + return _route("codex", "medium") + raise RuntimeError("Switchyard worker routing failed: refused") + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(1, "quota exceeded", None, True), + ) + + lanes.execute_claim("cassandra", "t_router_fb") + + assert selections == ["cli-auto", "cli-claude-medium"] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] + + +def test_goal_loop_router_outage_blocks_transient(tmp_path: Path, monkeypatch): + task = SimpleNamespace( + id="t_goal_router", + status="running", + result=None, + current_run_id=36, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + selections: list = [] + + def select_route(_prompt, lane, **_kwargs): + selections.append(lane) + if len(selections) == 1: + return _route("codex", "high") + raise RuntimeError("Switchyard worker routing failed: refused") + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "turn", dict(COMPLETED_RESULT), False + ), + ) + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + lambda *_args, **_kwargs: (False, "verification is still missing"), + ) + continued: list = [] + monkeypatch.setattr( + lanes, + "record_route_decision", + lambda provider, _effort, _classifier, outcome: continued.append( + (provider, outcome) + ), + ) + + lanes.execute_claim("cassandra", "t_goal_router") + + assert selections == ["cli-auto", "cli-auto"] + assert ("codex", "goal-continued") in continued + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] + + +def test_soft_quota_exclusion_steers_new_auto_work(tmp_path: Path, monkeypatch): + task = SimpleNamespace( + id="t_quota", + status="running", + result=None, + current_run_id=37, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + monkeypatch.setattr( + lanes, + "fetch_quota_snapshot", + lambda *_a, **_k: { + "codex": lanes.ProviderQuota(10.0, None), + "claude": lanes.ProviderQuota(60.0, None), + }, + ) + route_calls: list = [] + monkeypatch.setattr( + lanes, + "select_route", + lambda _prompt, lane, **kwargs: ( + route_calls.append((lane, kwargs)) or _route("claude", "high") + ), + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "done", dict(COMPLETED_RESULT), False + ), + ) + + lanes.execute_claim("cassandra", "t_quota") + + assert calls and calls[0][0] == "complete" + lane, kwargs = route_calls[0] + assert lane == "cli-auto" + assert kwargs["exclude_provider"] == "codex" + assert kwargs["exclude_reason"] == "is below its remaining-quota routing threshold" + assert any("Quota guard: codex" in item for item in comments) + assert any( + "Provider routing guard (quota) excluded codex" in item for item in comments + ) diff --git a/testing/tests/test_hermes_cli_quota_gating.py b/testing/tests/test_hermes_cli_quota_gating.py new file mode 100644 index 00000000..c3ce7fa7 --- /dev/null +++ b/testing/tests/test_hermes_cli_quota_gating.py @@ -0,0 +1,299 @@ +"""Quota-aware soft routing thresholds, parsing, and manifest wiring.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_support import ( + HERMES, + Path, + lanes, + yaml, +) + + +EXPORTER_TEXT = """\ +# HELP atlas_ai_quota_remaining_percent Remaining percentage. +# TYPE atlas_ai_quota_remaining_percent gauge +atlas_ai_quota_remaining_percent{limit="codex",provider="openai",window="five_hour"} 37.5 +atlas_ai_quota_remaining_percent{limit="codex",provider="openai",window="seven_day"} 80 +atlas_ai_quota_remaining_percent{limit="bonus-credits",provider="openai",window="five_hour"} 1 +atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="five_hour"} 62 +atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="seven_day"} 40 +atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="seven_day_opus"} 1 +atlas_ai_quota_remaining_percent{limit="other",provider="somebody",window="five_hour"} 2 +atlas_ai_quota_reset_timestamp_seconds{limit="codex",provider="openai",window="five_hour"} 111 +atlas_ai_quota_reset_timestamp_seconds{limit="codex",provider="openai",window="seven_day"} 222 +atlas_ai_quota_reset_timestamp_seconds{limit="claude",provider="anthropic",window="seven_day"} 444 +atlas_ai_quota_fetch_success{provider="openai"} 1 +atlas_ai_quota_fetch_success{provider="anthropic"} 1 +not a metric line at all +""" + + +class _MetricsResponse: + def __init__(self, text: str): + self._payload = text.encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _limit: int) -> bytes: + return self._payload + + +def test_parse_reduces_each_provider_to_its_binding_account_window(): + snapshot = lanes.parse_quota_metrics(EXPORTER_TEXT) + + assert snapshot["codex"].remaining_percent == 37.5 + assert snapshot["codex"].reset_timestamp == 111 + # The model-specific seven_day_opus window must not veto claude; the + # binding account window is seven_day at 40% with its own reset. + assert snapshot["claude"].remaining_percent == 40 + assert snapshot["claude"].reset_timestamp == 444 + + +def test_parse_drops_provider_after_failed_exporter_fetch(): + text = EXPORTER_TEXT.replace( + 'atlas_ai_quota_fetch_success{provider="openai"} 1', + 'atlas_ai_quota_fetch_success{provider="openai"} 0', + ) + snapshot = lanes.parse_quota_metrics(text) + + assert "codex" not in snapshot + assert snapshot["claude"].remaining_percent == 40 + + +def test_parse_reports_no_reset_when_binding_window_lacks_one(): + text = ( + 'atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="five_hour"} 9\n' + ) + snapshot = lanes.parse_quota_metrics(text) + + assert snapshot == {"claude": lanes.ProviderQuota(9.0, None)} + + +def test_fetch_snapshot_fails_open_on_transport_error(monkeypatch, capsys): + failures = [] + monkeypatch.setattr(lanes, "record_quota_fetch_failure", lambda: failures.append(1)) + + def refused(_url, timeout): + raise OSError("connection refused") + + assert lanes.fetch_quota_snapshot(open_url=refused) == {} + assert failures == [1] + assert "fails open" in capsys.readouterr().err + + +def test_fetch_snapshot_counts_empty_signal_and_parses_live_payload( + monkeypatch, capsys +): + failures = [] + monkeypatch.setattr(lanes, "record_quota_fetch_failure", lambda: failures.append(1)) + + empty = lanes.fetch_quota_snapshot( + open_url=lambda _url, timeout: _MetricsResponse("") + ) + assert empty == {} + assert failures == [1] + assert "no gated windows" in capsys.readouterr().err + + live = lanes.fetch_quota_snapshot( + open_url=lambda _url, timeout: _MetricsResponse(EXPORTER_TEXT) + ) + assert live["codex"].remaining_percent == 37.5 + assert failures == [1] + + +def test_threshold_boundary_is_strictly_below(): + for remaining, expected in ((14.9, "codex"), (15.0, None), (15.1, None)): + snapshot = { + "codex": lanes.ProviderQuota(remaining, None), + "claude": lanes.ProviderQuota(80.0, None), + } + excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0) + assert excluded == expected + if expected is None: + assert note is None + else: + assert "14.9% remaining" in note + assert "threshold 15%" in note + + +def test_both_providers_below_threshold_prefers_more_remaining(): + snapshot = { + "codex": lanes.ProviderQuota(3.0, None), + "claude": lanes.ProviderQuota(9.0, None), + } + excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0) + + assert excluded == "codex" + assert "every provider is below" in note + assert "preferring claude (9% left) over codex (3% left)" in note + + +def test_both_below_tie_excludes_deterministically(): + snapshot = { + "codex": lanes.ProviderQuota(5.0, None), + "claude": lanes.ProviderQuota(5.0, None), + } + excluded, _note = lanes.quota_soft_exclusion(snapshot, 15.0) + + assert excluded == "claude" + + +def test_single_known_provider_below_threshold_is_excluded(): + snapshot = {"claude": lanes.ProviderQuota(2.0, None)} + excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0) + + assert excluded == "claude" + assert "routing new auto work" in note + + +def test_kanban_setting_reads_deployed_config_and_fails_to_default( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + config = tmp_path / "config.yaml" + config.write_text("kanban:\n provider_quota_min_remaining_percent: 22\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 22.0 + + config.write_text("kanban: 5\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + config.write_text("just a string\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + config.write_text("kanban:\n provider_quota_min_remaining_percent: true\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + config.write_text("kanban:\n provider_quota_min_remaining_percent: '20'\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + config.write_text("kanban: [\n") + assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0 + + +def test_selection_constraint_never_gates_explicit_lanes(): + def forbidden(): + raise AssertionError("explicit lanes must not read quota state") + + constraint = lanes.selection_constraint("cli-codex-high", snapshot_from=forbidden) + + assert constraint == lanes.SelectionConstraint(None, None, None, ()) + + +def test_selection_constraint_health_exclusion_wins_and_carries_resets(monkeypatch): + observed = {} + + def health(now=None, quota_resets=None): + observed["resets"] = quota_resets + return "codex" + + monkeypatch.setattr(lanes, "fresh_unavailable_provider", health) + snapshot = {"claude": lanes.ProviderQuota(50.0, 1234.0)} + constraint = lanes.selection_constraint( + "cli-auto", snapshot_from=lambda: snapshot + ) + + assert constraint.exclude_provider == "codex" + assert constraint.source == "health" + assert constraint.notes == () + assert observed["resets"] == {"claude": 1234.0} + + +def test_selection_constraint_quota_gate_publishes_gauges( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) + monkeypatch.setattr( + lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None + ) + snapshot = { + "codex": lanes.ProviderQuota(10.0, 555.0), + "claude": lanes.ProviderQuota(60.0, None), + } + constraint = lanes.selection_constraint("cli-auto", snapshot_from=lambda: snapshot) + + assert constraint.exclude_provider == "codex" + assert constraint.source == "quota" + assert constraint.exclude_reason == "is below its remaining-quota routing threshold" + assert len(constraint.notes) == 1 and "Quota guard: codex" in constraint.notes[0] + rendered = lanes.METRICS.render().decode("utf-8") + assert 'hermes_cli_quota_remaining_percent{provider="codex"} 10' in rendered + assert 'hermes_cli_quota_reset_timestamp_seconds{provider="codex"} 555' in rendered + assert 'hermes_cli_provider_soft_excluded{provider="codex"} 1' in rendered + assert 'hermes_cli_provider_soft_excluded{provider="claude"} 0' in rendered + + +def test_selection_constraint_fails_open_without_quota_signal( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) + monkeypatch.setattr( + lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None + ) + constraint = lanes.selection_constraint("cli-auto", snapshot_from=dict) + + assert constraint == lanes.SelectionConstraint(None, None, None, ()) + + +def _agent_config_kanban() -> dict: + document = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + return yaml.safe_load(document["data"]["config.yaml"])["kanban"] + + +def test_configmap_declares_quota_routing_defaults(): + kanban = _agent_config_kanban() + + assert kanban["provider_quota_min_remaining_percent"] == 15 + assert kanban["provider_capacity_cooldown_seconds"] == 300 + assert kanban["provider_auth_cooldown_seconds"] == 3600 + + +def test_lane_metrics_are_wired_for_scraping(): + deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + containers = deployment["spec"]["template"]["spec"]["containers"] + lane = next(item for item in containers if item["name"] == "cli-lane-runner") + env = {entry["name"]: entry.get("value") for entry in lane["env"]} + + assert {"name": "lane-metrics", "containerPort": 9011, "protocol": "TCP"} in lane[ + "ports" + ] + assert env["HERMES_CLI_LANE_METRICS_PORT"] == "9011" + assert env["HERMES_CLI_QUOTA_METRICS_URL"] == "http://127.0.0.1:9010/metrics" + + services = { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + metrics_service = services["hermes-cli-lane-metrics"] + annotations = metrics_service["metadata"]["annotations"] + assert annotations["prometheus.io/scrape"] == "true" + assert annotations["prometheus.io/port"] == "9011" + assert metrics_service["spec"]["selector"] == {"app": "hermes-agent"} + assert metrics_service["spec"]["ports"][0]["targetPort"] == "lane-metrics" + + policies = { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + } + agent_ingress = policies["hermes-agent-isolation"]["spec"]["ingress"] + monitoring_ports = [ + port["port"] + for rule in agent_ingress + for port in rule.get("ports", []) + if any( + source.get("namespaceSelector", {}) + .get("matchLabels", {}) + .get("kubernetes.io/metadata.name") + == "monitoring" + for source in rule.get("from", []) + ) + ] + assert 9010 in monitoring_ports and 9011 in monitoring_ports From 6e9e8c6ab5e5cc560e45a606c47ab83b8c261d64 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 38/46] hermes: reject regex literals in polkit grants A regex literal embedding a quote desynced the string tokenizer and swallowed an unconditional Result.YES, so the grant read as unscoped and slipped past the literal-identity check. Fail closed on any slash outside a string or comment; neither regex nor division belongs in an identity-scoped grant. Co-Authored-By: Claude Fable 5 --- services/hermes/scripts/node_polkit_audit.py | 7 +++ .../tests/test_hermes_node_polkit_identity.py | 56 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py index 281f1484..d45a0134 100644 --- a/services/hermes/scripts/node_polkit_audit.py +++ b/services/hermes/scripts/node_polkit_audit.py @@ -85,6 +85,13 @@ def _split_strings(text: str) -> tuple[str, list[str]]: code.append(" ") index = closing + 2 continue + if character == "/": + # Any remaining slash is a regex literal or a division operator. + # A regex literal may embed a quote character, which would desync + # the string scanner and swallow an unconditional grant. Neither + # regex nor division has a place in an identity-scoped grant, so + # both fail closed before they can hide a Result.YES. + raise HardeningError("polkit rule uses a regex or division operator") code.append(character) index += 1 return "".join(code), strings diff --git a/testing/tests/test_hermes_node_polkit_identity.py b/testing/tests/test_hermes_node_polkit_identity.py index 059f05f2..3af8d095 100644 --- a/testing/tests/test_hermes_node_polkit_identity.py +++ b/testing/tests/test_hermes_node_polkit_identity.py @@ -124,6 +124,62 @@ def test_unprovable_rule_constructs_fail_closed(text: str, match: str): _rules(module, text) +def test_regex_literal_tokenizer_desync_grant_is_rejected(): + """A regex literal embedding a quote must not open a phantom string. + + Without regex awareness the two /'/ literals fool the string scanner into + swallowing the unconditional Result.YES between their quotes; the grant + then reads as unscoped and would slip past every later check. + """ + module = _load() + exploit = ( + "polkit.addRule(function(action, subject) { " + "var a = /'/; " + 'if (action.id == "org.freedesktop.policykit.exec") ' + "{ return polkit.Result.YES; } " + "var b = /'/; });\n" + ) + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, exploit) + + +@pytest.mark.parametrize( + "snippet", + [ + "var a = /'/;", + 'var a = /"/;', + "var a = /abc/;", + "var a = /[/'\"]/;", + "var a = /x\\/y/;", + "var a = subject.user / 2;", + "// a comment / with a slash then\nvar a = /'/;", + '/* block */ var a = /"/;', + "return polkit.Result.YES / 1;", + ], +) +def test_every_slash_form_outside_a_comment_fails_closed(snippet: str): + module = _load() + text = f"polkit.addRule(function(action, subject) {{ {snippet} }});\n" + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, text) + + +def test_slashes_confined_to_strings_and_comments_stay_legal(): + module = _load() + # Slashes inside string literals and // or /* */ comments are inert. + _rules( + module, + "// path-like /usr/bin comment\n" + "/* another /etc/ comment */\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.systemd1.manage-units" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + def test_string_and_comment_handling_keeps_literal_rules_auditable(): module = _load() code, strings = module._split_strings( From 6c1123201e7ce642a5fc1c89d014a97a9bd3f0ef Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 39/46] hermes: audit every sudo include spelling A quoted, space-bearing #include path failed the include regex and fell through to the comment branch, leaving a second authority file unenumerated. Detect any include directive before the comment rule and fail closed on every form except the exact bare includedir into the audited /etc/sudoers.d. Co-Authored-By: Claude Fable 5 --- services/hermes/scripts/node_account_audit.py | 23 +++++++--- .../tests/test_hermes_node_audit_coverage.py | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py index 31b54302..fa6245ba 100644 --- a/services/hermes/scripts/node_account_audit.py +++ b/services/hermes/scripts/node_account_audit.py @@ -89,13 +89,22 @@ def _active_sudo_policy(text: str) -> str: stripped = line.lstrip() if not stripped: continue - include = re.fullmatch( - r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped - ) - if include: - directive, target = include.groups() - if directive.lower().endswith("includedir") and target == "/etc/sudoers.d": - continue + # Detect any include spelling before the comment rule below. sudo + # honours both #include and @include; a quoted or space-bearing path + # must never fall through the "#" comment branch and leave a second + # authority file unaudited. Only the exact bare includedir into the + # audited /etc/sudoers.d passes; everything else fails closed. + if re.match(r"(?i)[#@]include(?:dir)?(?=\s|$)", stripped): + include = re.fullmatch( + r"(?i)([#@]include(?:dir)?)\s+([^\s\"'\\]+)", stripped + ) + if include: + directive, target = include.groups() + if ( + directive.lower().endswith("includedir") + and target == "/etc/sudoers.d" + ): + continue raise HardeningError("sudo policy includes an unaudited authority source") if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): continue diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py index 8c011b5b..20114ada 100644 --- a/testing/tests/test_hermes_node_audit_coverage.py +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -105,6 +105,50 @@ def test_active_sudo_policy_handles_blank_comments_numeric_and_includes(): module._active_sudo_policy("@include /external/policy\n") +@pytest.mark.parametrize( + "directive", + [ + '#include "/etc/evil path"', + "#include '/etc/evil path'", + "#include /etc/evil\\ path", + "#include\t/etc/evil", + "#include /etc/evil", + "#includedir /opt/other", + '#includedir "/etc/sudoers.d"', + "@include /external/policy", + '@include "/etc/evil path"', + "@includedir /opt/other", + "#include", + "@includedir", + ], +) +def test_every_include_spelling_outside_the_standard_dir_fails_closed(directive: str): + """No include form may reach the comment branch and stay unaudited.""" + module = _audit_module() + with pytest.raises(module.HardeningError, match="unaudited"): + module._active_sudo_policy(directive + "\n") + + +@pytest.mark.parametrize( + "directive", + ["#includedir /etc/sudoers.d", "@includedir /etc/sudoers.d"], +) +def test_standard_includedir_is_the_only_allowed_include(directive: str): + module = _audit_module() + active = module._active_sudo_policy(directive + "\nroot ALL=(ALL) ALL\n") + assert active == "root ALL=(ALL) ALL" + + +@pytest.mark.parametrize( + "line", + ["#included by an operator note", "#include-guard notes", "#includes history"], +) +def test_comment_lines_that_only_resemble_includes_stay_comments(line: str): + module = _audit_module() + active = module._active_sudo_policy(line + "\nroot ALL=(ALL) ALL\n") + assert active == "root ALL=(ALL) ALL" + + def test_identity_matcher_covers_account_numeric_and_unrelated_values(): module = _audit_module() assert module._mentions_dedicated_identity("hermes-agent ALL", "hermes-agent", 1200) From 31439f096afb0acba710d4d0ebb4b52a84a3b370 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 40/46] hermes: bound streaming git DNS resolution The watchdog can only close a live socket, but getaddrinfo runs before any socket exists and ignores socket timeouts, so a slow resolver outlived the stream deadline. Resolve under the same absolute deadline in a joinable worker before the real connect. Co-Authored-By: Claude Fable 5 --- .../scm-common/scripts/deadline_http.py | 46 +++++++++- testing/tests/test_hermes_deadline_stream.py | 87 +++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py index a19abccc..e513a446 100644 --- a/services/hermes/scm-common/scripts/deadline_http.py +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -227,6 +227,31 @@ def open_bounded( ) +def _resolve_within(host: str, port: int, timeout: float) -> None: + """Resolve one host under a hard timeout the socket layer never covers. + + ``getaddrinfo`` ignores socket timeouts and runs before any socket + exists, so a slow or hostile resolver would otherwise outlive the whole + stream deadline. Running it in a joinable worker bounds resolution by the + same wall clock; a stuck lookup leaks only one daemon thread. + """ + outcome: dict[str, object] = {} + + def run() -> None: + try: + outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(timeout) + if worker.is_alive(): + raise PolicyError("HTTP stream DNS resolution deadline exceeded") + if "error" in outcome: + raise outcome["error"] # type: ignore[misc] + + class StreamDeadline: """Force-close tracked connections once an absolute deadline passes.""" @@ -270,6 +295,23 @@ class StreamDeadline: raise PolicyError("HTTP stream deadline exceeded") return connection + def _bounded_connection(self, connection: http.client.HTTPConnection): + """Register one connection and bound its DNS phase by the deadline. + + The watchdog can only close a live socket, but ``connect`` resolves + the host before any socket exists. Wrapping ``connect`` runs that + resolution under the same absolute deadline first, then hands off to + the real connect for the socket phases the watchdog already covers. + """ + original_connect = connection.connect + + def connect() -> None: + _resolve_within(connection.host, connection.port, self.remaining()) + original_connect() + + connection.connect = connect # type: ignore[method-assign] + return self._track(connection) + def cancel(self) -> None: self._timer.cancel() @@ -280,7 +322,7 @@ class StreamDeadline: class GuardedHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open( - lambda host, **kwargs: deadline._track( + lambda host, **kwargs: deadline._bounded_connection( http.client.HTTPConnection(host, **kwargs) ), req, @@ -289,7 +331,7 @@ class StreamDeadline: class GuardedHTTPSHandler(urllib.request.HTTPSHandler): def https_open(self, req): return self.do_open( - lambda host, **kwargs: deadline._track( + lambda host, **kwargs: deadline._bounded_connection( http.client.HTTPSConnection(host, **kwargs) ), req, diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py index d4f97efe..88022591 100644 --- a/testing/tests/test_hermes_deadline_stream.py +++ b/testing/tests/test_hermes_deadline_stream.py @@ -107,6 +107,93 @@ def test_handlers_build_guarded_connections_without_network(monkeypatch): guard.cancel() +def test_dns_resolution_is_bounded_by_the_stream_deadline(monkeypatch): + module = _load("deadline_http") + + def hang(*_args, **_kwargs): + time.sleep(30) + + monkeypatch.setattr(module.socket, "getaddrinfo", hang) + started = time.monotonic() + with pytest.raises(module.PolicyError, match="DNS resolution deadline"): + module._resolve_within("example.invalid", 443, 0.4) + assert time.monotonic() - started < 3 + + +def test_resolve_within_returns_on_success_and_propagates_errors(monkeypatch): + module = _load("deadline_http") + calls = [] + monkeypatch.setattr( + module.socket, "getaddrinfo", lambda *a, **k: calls.append((a, k)) + ) + module._resolve_within("127.0.0.1", 80, 5) + assert calls + + def boom(*_a, **_k): + raise OSError("name resolution failed") + + monkeypatch.setattr(module.socket, "getaddrinfo", boom) + with pytest.raises(OSError, match="name resolution failed"): + module._resolve_within("127.0.0.1", 80, 5) + + +def test_bounded_connection_wraps_connect_to_resolve_first(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + order = [] + monkeypatch.setattr( + module, + "_resolve_within", + lambda host, port, timeout: order.append(("resolve", host, port)), + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + + def __init__(self): + self.sock = None + + def connect(self): + order.append(("connect", self.host)) + + def close(self): + order.append(("close",)) + + connection = guard._bounded_connection(FakeConnection()) + assert connection in guard._connections + connection.connect() + assert order == [("resolve", "scm.bstein.dev", 443), ("connect", "scm.bstein.dev")] + guard.cancel() + + +def test_bounded_connection_connect_fails_closed_once_expired(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + resolved = [] + monkeypatch.setattr( + module, "_resolve_within", lambda *a, **k: resolved.append(a) + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + sock = None + + def connect(self): + resolved.append("connected") + + def close(self): + pass + + connection = guard._bounded_connection(FakeConnection()) + guard._expire() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + connection.connect() + assert resolved == [] + guard.cancel() + + class _TrickleHandler(BaseHTTPRequestHandler): def do_GET(self): time.sleep(10) From a5d0abb67853753c0df782940154334b79ce9297 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:46:00 -0300 Subject: [PATCH 41/46] hermes: align capacity gate with auth classifier CAPACITY_PATTERN (the gate that sets result.capacity_failure) lacked the bare unauthorized/forbidden/401/403 signals that classify_capacity_failure already recognizes, so an auth blip surfacing only as "403 Forbidden" was blocked as capability instead of transient: no auto failover, no health cooldown. Add 401|403|unauthorized|forbidden to the gate so it matches the classifier; reason classification still distinguishes auth from quota/rate-limit/transport. Tests: an auto card failing with only "403 Forbidden" now fails over to the other provider, records an auth cooldown (authenticated:false), and classifies the fallback reason as auth in metrics; a manual card with the same failure still fails closed as transient. Router-outage tests moved to test_hermes_cli_router_outage.py to keep both files <500 LOC. Based on PR #15 (fix/hermes-result-decomposition-reliability). Co-Authored-By: Claude Opus 4.8 --- services/hermes/scripts/cli_lane_config.py | 3 +- .../tests/test_hermes_cli_quota_failover.py | 208 +++++++----------- .../tests/test_hermes_cli_router_outage.py | 149 +++++++++++++ 3 files changed, 225 insertions(+), 135 deletions(-) create mode 100644 testing/tests/test_hermes_cli_router_outage.py diff --git a/services/hermes/scripts/cli_lane_config.py b/services/hermes/scripts/cli_lane_config.py index d570b2ca..895aa5cc 100644 --- a/services/hermes/scripts/cli_lane_config.py +++ b/services/hermes/scripts/cli_lane_config.py @@ -55,7 +55,8 @@ WORKTREE_LOCK = threading.Lock() BOARD_CORRUPTION_ERRORS: dict[str, str] = {} LAST_ARTIFACT_GC = 0.0 CAPACITY_PATTERN = re.compile( - r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", + r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529" + r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)", re.I, ) NO_CLAUDE_SESSION = "No conversation found with session ID:" diff --git a/testing/tests/test_hermes_cli_quota_failover.py b/testing/tests/test_hermes_cli_quota_failover.py index 3cc54850..ff503609 100644 --- a/testing/tests/test_hermes_cli_quota_failover.py +++ b/testing/tests/test_hermes_cli_quota_failover.py @@ -196,6 +196,80 @@ def test_auto_failover_preserves_effort_and_records_cooldown( assert claude_health["state"] == "available" +def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_forbidden", + status="running", + result=None, + current_run_id=38, + assignee="cli-auto", + max_runtime_seconds=120, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + health_paths = _isolate_lane(tmp_path, monkeypatch, board) + + def select_route(_prompt, lane, **_kwargs): + return ( + _route("codex", "high") if lane == "cli-auto" else _route("claude", "high") + ) + + monkeypatch.setattr(lanes, "select_route", select_route) + fallbacks: list = [] + monkeypatch.setattr( + lanes, + "record_provider_fallback", + lambda source, target, reason: fallbacks.append((source, target, reason)), + ) + # The failure surfaces only as "403 Forbidden" with none of the words the + # capacity gate previously keyed on (authentication/oauth/token expired). + reports = [ + lanes.ProcessResult(1, "403 Forbidden", None, True), + lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + + lanes.execute_claim("cassandra", "t_forbidden") + + # (a) failed over to the other provider and completed. + assert reports == [] + assert calls and calls[0][0] == "complete" + # (c) the fallback reason classifies as auth in metrics. + assert fallbacks == [("codex", "claude", "auth")] + assert any( + "Provider fallback: codex -> claude after a auth failure" in item + for item in comments + ) + # (b) an auth cooldown was recorded (authenticated:false, longer window). + codex_health = json.loads(health_paths["codex"].read_text()) + assert codex_health["state"] == "unavailable" + assert codex_health["authenticated"] is False + assert codex_health["failure_reason"] == "auth" + assert codex_health["cooldown_until"] == codex_health["failed_at"] + 3600.0 + + +def test_bare_forbidden_auth_blip_still_fails_closed_on_manual_lane( + tmp_path: Path, monkeypatch +): + calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( + tmp_path, monkeypatch, "cli-codex-high", "codex", "403 Forbidden" + ) + + # Fail-closed is unchanged: the pinned provider is not switched. + assert runs == ["codex"] + assert [lane for lane, _ in route_calls] == ["cli-codex-high"] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Manually pinned provider codex" in kwargs["reason"] + assert "auth" in kwargs["reason"] + codex_health = json.loads(health_paths["codex"].read_text()) + assert codex_health["state"] == "unavailable" + assert codex_health["authenticated"] is False + + def test_double_capacity_failure_blocks_transient_with_both_reasons( tmp_path: Path, monkeypatch ): @@ -256,140 +330,6 @@ def test_double_capacity_failure_blocks_transient_with_both_reasons( assert claude_health["authenticated"] is False -def test_router_outage_blocks_transient_before_any_provider_run( - tmp_path: Path, monkeypatch -): - task = SimpleNamespace( - id="t_router", - status="running", - result=None, - current_run_id=34, - assignee="cli-auto", - max_runtime_seconds=60, - ) - comments: list = [] - calls: list = [] - board = _lane_board(tmp_path, task, comments, calls) - _isolate_lane(tmp_path, monkeypatch, board) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("Switchyard worker routing failed: refused") - ), - ) - outages: list = [] - monkeypatch.setattr( - lanes, "record_router_selection_failure", lambda: outages.append(1) - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - AssertionError("no provider may run without a route") - ), - ) - - lanes.execute_claim("cassandra", "t_router") - - assert outages == [1] - kind, kwargs = calls[-1] - assert kind == "block" and kwargs["kind"] == "transient" - assert "Switchyard route selection is unavailable" in kwargs["reason"] - - -def test_router_outage_during_fallback_selection_blocks_transient( - tmp_path: Path, monkeypatch -): - task = SimpleNamespace( - id="t_router_fb", - status="running", - result=None, - current_run_id=35, - assignee="cli-auto", - max_runtime_seconds=60, - ) - comments: list = [] - calls: list = [] - board = _lane_board(tmp_path, task, comments, calls) - _isolate_lane(tmp_path, monkeypatch, board) - selections: list = [] - - def select_route(_prompt, lane, **_kwargs): - selections.append(lane) - if lane == "cli-auto": - return _route("codex", "medium") - raise RuntimeError("Switchyard worker routing failed: refused") - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: lanes.ProcessResult(1, "quota exceeded", None, True), - ) - - lanes.execute_claim("cassandra", "t_router_fb") - - assert selections == ["cli-auto", "cli-claude-medium"] - kind, kwargs = calls[-1] - assert kind == "block" and kwargs["kind"] == "transient" - assert "Switchyard route selection is unavailable" in kwargs["reason"] - - -def test_goal_loop_router_outage_blocks_transient(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_goal_router", - status="running", - result=None, - current_run_id=36, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - comments: list = [] - calls: list = [] - board = _lane_board(tmp_path, task, comments, calls) - _isolate_lane(tmp_path, monkeypatch, board) - selections: list = [] - - def select_route(_prompt, lane, **_kwargs): - selections.append(lane) - if len(selections) == 1: - return _route("codex", "high") - raise RuntimeError("Switchyard worker routing failed: refused") - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr( - lanes, - "run_provider", - lambda *_args, **_kwargs: lanes.ProcessResult( - 0, "turn", dict(COMPLETED_RESULT), False - ), - ) - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - lambda *_args, **_kwargs: (False, "verification is still missing"), - ) - continued: list = [] - monkeypatch.setattr( - lanes, - "record_route_decision", - lambda provider, _effort, _classifier, outcome: continued.append( - (provider, outcome) - ), - ) - - lanes.execute_claim("cassandra", "t_goal_router") - - assert selections == ["cli-auto", "cli-auto"] - assert ("codex", "goal-continued") in continued - kind, kwargs = calls[-1] - assert kind == "block" and kwargs["kind"] == "transient" - assert "Switchyard route selection is unavailable" in kwargs["reason"] - - def test_soft_quota_exclusion_steers_new_auto_work(tmp_path: Path, monkeypatch): task = SimpleNamespace( id="t_quota", diff --git a/testing/tests/test_hermes_cli_router_outage.py b/testing/tests/test_hermes_cli_router_outage.py new file mode 100644 index 00000000..b2d7b01d --- /dev/null +++ b/testing/tests/test_hermes_cli_router_outage.py @@ -0,0 +1,149 @@ +"""Switchyard outage blocks a card as transient at every selection boundary.""" + +from __future__ import annotations + +from testing.tests.test_hermes_cli_quota_failover import ( + COMPLETED_RESULT, + _isolate_lane, + _lane_board, + _route, +) +from testing.tests.test_hermes_cli_support import ( + Path, + SimpleNamespace, + lanes, +) + + +def test_router_outage_blocks_transient_before_any_provider_run( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_router", + status="running", + result=None, + current_run_id=34, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("Switchyard worker routing failed: refused") + ), + ) + outages: list = [] + monkeypatch.setattr( + lanes, "record_router_selection_failure", lambda: outages.append(1) + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("no provider may run without a route") + ), + ) + + lanes.execute_claim("cassandra", "t_router") + + assert outages == [1] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] + + +def test_router_outage_during_fallback_selection_blocks_transient( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_router_fb", + status="running", + result=None, + current_run_id=35, + assignee="cli-auto", + max_runtime_seconds=60, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + selections: list = [] + + def select_route(_prompt, lane, **_kwargs): + selections.append(lane) + if lane == "cli-auto": + return _route("codex", "medium") + raise RuntimeError("Switchyard worker routing failed: refused") + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult(1, "quota exceeded", None, True), + ) + + lanes.execute_claim("cassandra", "t_router_fb") + + assert selections == ["cli-auto", "cli-claude-medium"] + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] + + +def test_goal_loop_router_outage_blocks_transient(tmp_path: Path, monkeypatch): + task = SimpleNamespace( + id="t_goal_router", + status="running", + result=None, + current_run_id=36, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + _isolate_lane(tmp_path, monkeypatch, board) + selections: list = [] + + def select_route(_prompt, lane, **_kwargs): + selections.append(lane) + if len(selections) == 1: + return _route("codex", "high") + raise RuntimeError("Switchyard worker routing failed: refused") + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr( + lanes, + "run_provider", + lambda *_args, **_kwargs: lanes.ProcessResult( + 0, "turn", dict(COMPLETED_RESULT), False + ), + ) + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + lambda *_args, **_kwargs: (False, "verification is still missing"), + ) + continued: list = [] + monkeypatch.setattr( + lanes, + "record_route_decision", + lambda provider, _effort, _classifier, outcome: continued.append( + (provider, outcome) + ), + ) + + lanes.execute_claim("cassandra", "t_goal_router") + + assert selections == ["cli-auto", "cli-auto"] + assert ("codex", "goal-continued") in continued + kind, kwargs = calls[-1] + assert kind == "block" and kwargs["kind"] == "transient" + assert "Switchyard route selection is unavailable" in kwargs["reason"] From 77df54e3ba9641de6aad8f932428cdb02ed150c7 Mon Sep 17 00:00:00 2001 From: flux-bot Date: Tue, 18 Aug 2026 01:39:17 +0000 Subject: [PATCH 42/46] chore(maintenance): automated image update --- services/maintenance/apps/metis-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/maintenance/apps/metis-configmap.yaml b/services/maintenance/apps/metis-configmap.yaml index 810d32cc..48e6a2aa 100644 --- a/services/maintenance/apps/metis-configmap.yaml +++ b/services/maintenance/apps/metis-configmap.yaml @@ -15,7 +15,7 @@ data: METIS_MAX_DEVICE_BYTES: "1000000000000" METIS_NAMESPACE: maintenance METIS_REMOTE_POD_TIMEOUT_SEC: "14400" - METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} + METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} METIS_HARBOR_REGISTRY: registry.bstein.dev METIS_HARBOR_PROJECT: metis From 376cc14dd19a29c8bfae6b3938b8fedda1efe5c6 Mon Sep 17 00:00:00 2001 From: flux-bot Date: Tue, 18 Aug 2026 01:44:19 +0000 Subject: [PATCH 43/46] chore(maintenance): automated image update --- .../maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml index 691ebd94..431af572 100644 --- a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: amd64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} imagePullPolicy: Always envFrom: - configMapRef: From 6b1133152093226e7a701002f0c70966930dd552 Mon Sep 17 00:00:00 2001 From: flux-bot Date: Tue, 18 Aug 2026 01:47:20 +0000 Subject: [PATCH 44/46] chore(maintenance): automated image update --- services/maintenance/apps/metis-configmap.yaml | 2 +- services/maintenance/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/maintenance/apps/metis-configmap.yaml b/services/maintenance/apps/metis-configmap.yaml index 48e6a2aa..36979463 100644 --- a/services/maintenance/apps/metis-configmap.yaml +++ b/services/maintenance/apps/metis-configmap.yaml @@ -16,7 +16,7 @@ data: METIS_NAMESPACE: maintenance METIS_REMOTE_POD_TIMEOUT_SEC: "14400" METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} - METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} + METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} METIS_HARBOR_REGISTRY: registry.bstein.dev METIS_HARBOR_PROJECT: metis METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0 diff --git a/services/maintenance/kustomization.yaml b/services/maintenance/kustomization.yaml index 5a2b82b0..86d52b56 100644 --- a/services/maintenance/kustomization.yaml +++ b/services/maintenance/kustomization.yaml @@ -13,6 +13,6 @@ images: - name: registry.bstein.dev/bstein/ariadne newTag: 0.1.0-461 # {"$imagepolicy": "maintenance:ariadne:tag"} - name: registry.bstein.dev/bstein/metis - newTag: 0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} + newTag: 0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} - name: registry.bstein.dev/bstein/soteria newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"} From 8c7074ae16666aec94739f925ec6a89efb02b62b Mon Sep 17 00:00:00 2001 From: flux-bot Date: Tue, 18 Aug 2026 01:48:20 +0000 Subject: [PATCH 45/46] chore(maintenance): automated image update --- .../maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml index 21d909a6..6616e07e 100644 --- a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: arm64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} imagePullPolicy: Always envFrom: - configMapRef: From 5f27e50c74558b27abbb09e159a1c9c4c2f04c85 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 18 Aug 2026 02:00:19 -0300 Subject: [PATCH 46/46] hermes: reconcile pool tests and staging hygiene with the train API Post-merge fixes after rebasing the distributed worker pool onto the review-goal-semantics train tip: - Pool SCM submission tests target the train's relocated receive-pack scanner: FEATURE_REF_RE now lives in receive_pack_scan, bodies are built via the shared _receive_command helper (valid pack), and the update-rejection assertion matches the train's message. - Take the train's canonical test_hermes_scm_broker, test_hermes_cli_dispatch_runtime and test_hermes_cli_execution_edges, which exercise the train's broker/dispatch/execution behavior. - Runtime staging tests patch os.fchown alongside os.chown so the UID-10000 _write_secret path passes under a non-root gate runner (production ownership behavior unchanged). - Split the jenkins build-evidence contracts out of test_hermes_runtime_access into test_hermes_runtime_evidence to keep both files under the 500-LOC hygiene ceiling after the merge. Co-Authored-By: Claude Opus 4.8 --- .../tests/test_hermes_cli_dispatch_runtime.py | 37 ------- .../tests/test_hermes_cli_execution_edges.py | 15 +-- .../test_hermes_execution_pool_submission.py | 17 ++- testing/tests/test_hermes_runtime_access.py | 78 +------------ testing/tests/test_hermes_runtime_evidence.py | 104 ++++++++++++++++++ .../test_hermes_runtime_stage_coverage.py | 1 + testing/tests/test_hermes_scm_broker.py | 11 +- 7 files changed, 116 insertions(+), 147 deletions(-) create mode 100644 testing/tests/test_hermes_runtime_evidence.py diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py index 0593efcb..3ffc0b39 100644 --- a/testing/tests/test_hermes_cli_dispatch_runtime.py +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -14,8 +14,6 @@ from testing.tests.test_hermes_cli_support import SCRIPTS, lanes def test_board_slug_and_connection_failure_paths(monkeypatch): - assert lanes._owns_local_workspace(SimpleNamespace(workspace_path=" /workspace ")) - assert not lanes._owns_local_workspace(SimpleNamespace(workspace_path="")) assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra" assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback" failures = [] @@ -103,41 +101,6 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc assert lanes.claim_ready(set(), 1) == [] -def test_dispatch_skips_unhealthy_ineligible_and_unclaimed_entries(monkeypatch): - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - tasks = [ - SimpleNamespace(id="t_ineligible", status="ready", assignee="cli-auto"), - SimpleNamespace(id="t_unclaimed", status="ready", assignee="cli-auto"), - ] - db = SimpleNamespace( - list_boards=lambda **_kwargs: [ - {"slug": "offline"}, - {"slug": "cassandra"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: (_ for _ in ()).throw(OSError("offline")) - if board == "offline" - else Connection(board), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: tasks, - claim_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) - assert lanes.claim_ready( - set(), - 2, - eligible=lambda _board, task: task.id == "t_unclaimed", - ) == [] - monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) - lanes.recover_orphans() - - class _Future: def __init__(self): self.polls = 0 diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py index 42ad6e4c..d1ffbda8 100644 --- a/testing/tests/test_hermes_cli_execution_edges.py +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -16,13 +16,6 @@ class _Connection: return None -def test_goal_rejection_history_is_list_bounded(): - execution = __import__("cli_lane_execution") - history = ["first"] - assert execution._goal_rejections({"goal_rejections": history}) is history - assert execution._goal_rejections({"goal_rejections": "invalid"}) == [] - - def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch): db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), @@ -71,18 +64,12 @@ def test_transient_callback_storage_errors_do_not_kill_provider( def provider(*args, **_kwargs): assert args[6]("still working") is True - return lanes.ProcessResult( - 1, - "failed", - {"status": "incomplete", "blockers": ["synthetic provider failure"]}, - False, - ) + return lanes.ProcessResult(1, "failed", None, False) monkeypatch.setattr(lanes, "run_provider", provider) lanes.execute_claim("cassandra", "t_callbacks") assert blocks and blocks[0]["kind"] == "capability" - assert blocks[0]["reason"] == "synthetic provider failure" def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch): diff --git a/testing/tests/test_hermes_execution_pool_submission.py b/testing/tests/test_hermes_execution_pool_submission.py index 2ea26692..69df8c53 100644 --- a/testing/tests/test_hermes_execution_pool_submission.py +++ b/testing/tests/test_hermes_execution_pool_submission.py @@ -23,7 +23,9 @@ sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] import execution_pool_client as client # noqa: E402 import execution_pool_protocol as protocol # noqa: E402 import execution_pool_scm as scm # noqa: E402 +import receive_pack_scan # noqa: E402 import scm_broker # noqa: E402 +from testing.tests.test_hermes_scm_broker_support import _receive_command # noqa: E402 from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402 KEY, RESULT, @@ -50,7 +52,7 @@ def test_submission_refs_are_creation_only_and_attempt_scoped(): # Every candidate stays inside a reviewed namespace the broker accepts, and # none of them is a protected or base ref. for candidate in first + retry: - assert scm_broker.FEATURE_REF_RE.fullmatch(f"refs/heads/{candidate}") + assert receive_pack_scan.FEATURE_REF_RE.fullmatch(f"refs/heads/{candidate}") assert candidate not in {"main", "master"} @@ -65,15 +67,10 @@ def test_submission_refs_drop_oversized_names_and_fail_closed_when_empty(): def test_the_broker_accepts_the_attempt_ref_as_a_creation_and_still_refuses_updates(): zero, head = b"0" * 40, b"1" * 40 ref = b"refs/heads/feature/pool-attempt-2" - create = b" ".join((zero, head, ref)) - scm_broker._validate_receive_pack( - b"%04x%s\n0000" % (len(create) + 5, create), "token" - ) - update = b" ".join((b"2" * 40, head, ref)) - with pytest.raises(scm_broker.PolicyError, match="new namespaced branch"): - scm_broker._validate_receive_pack( - b"%04x%s\n0000" % (len(update) + 5, update), "token" - ) + scm_broker._validate_receive_pack(_receive_command(zero, head, ref), "token") + update = _receive_command(b"2" * 40, head, ref) + with pytest.raises(scm_broker.PolicyError, match="feature-branch creation"): + scm_broker._validate_receive_pack(update, "token") def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1): diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 42049146..68644623 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -82,82 +82,6 @@ def test_gitea_api_rejects_unavailable_methods(method: str): ) -def test_jenkins_evidence_ignores_nested_execution_result(): - evidence_reader = _load("jenkins_build_evidence") - evidence = evidence_reader.parse_build_xml( - """ - - abc123 - 10000 - SUCCESS - - """, - job="demo", - branch="master", - number=15, - ) - - assert evidence.revision == "abc123" - assert evidence.result is None - assert evidence.building is True - - -def test_jenkins_evidence_reads_only_top_level_terminal_result(): - evidence_reader = _load("jenkins_build_evidence") - evidence = evidence_reader.parse_build_xml( - """ - - abc123 - 1000250 - FAILURE - SUCCESS - - """, - job="demo", - branch=None, - number=4, - ) - - assert evidence.result == "FAILURE" - assert evidence.building is False - assert evidence.duration_ms == 250 - - -def test_jenkins_evidence_removes_hidden_links_and_ansi_control_sequences(): - evidence_reader = _load("jenkins_build_evidence") - - value = ( - "before\n\x1b[8mha:////private-jenkins-payload\x1b[0m[Pipeline] sh\n" - "\x1b[31m5 passed\x1b[0m\n" - ) - - assert evidence_reader.sanitize_log(value) == "before\n[Pipeline] sh\n5 passed\n" - - -def test_jenkins_evidence_falls_back_to_multibranch_suffix(monkeypatch): - evidence_reader = _load("jenkins_build_evidence") - calls = [] - - def fake_find(job, branch, commit, log_lines): - calls.append((job, branch, commit, log_lines)) - if job == "demo": - raise ValueError("not a multibranch job") - return "evidence" - - monkeypatch.setattr(evidence_reader, "_find_build_for_job", fake_find) - - assert evidence_reader.find_build("demo", "master", "abc", 20) == "evidence" - assert [call[0] for call in calls] == ["demo", "demo-branches"] - - -@pytest.mark.parametrize("branch", ["../master", "/master", "feature//unsafe"]) -def test_jenkins_evidence_rejects_unsafe_branch_paths(branch: str): - evidence_reader = _load("jenkins_build_evidence") - - with pytest.raises(ValueError): - evidence_reader._validate_branch(branch) - - def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeypatch): stage = _load("stage_runtime_access") vault = tmp_path / "vault" @@ -196,6 +120,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) monkeypatch.setattr(stage, "PERSISTENT_HOME", home) monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) stage.stage_agent() @@ -306,6 +231,7 @@ def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): (vault / "credential").write_text("not-json\n", encoding="utf-8") monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) destination = runtime / "credential.json" try: diff --git a/testing/tests/test_hermes_runtime_evidence.py b/testing/tests/test_hermes_runtime_evidence.py new file mode 100644 index 00000000..11ac9bc0 --- /dev/null +++ b/testing/tests/test_hermes_runtime_evidence.py @@ -0,0 +1,104 @@ +"""Contracts for Hermes' Jenkins build-evidence reader boundary.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" +SCRIPTS = HERMES / "scripts" +SCM_SCRIPTS = HERMES / "scm-common" / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) +if str(SCM_SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCM_SCRIPTS)) + + +def _load(name: str): + root = SCM_SCRIPTS if name == "gitea_api" else SCRIPTS + spec = importlib.util.spec_from_file_location(name, root / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_jenkins_evidence_ignores_nested_execution_result(): + evidence_reader = _load("jenkins_build_evidence") + evidence = evidence_reader.parse_build_xml( + """ + + abc123 + 10000 + SUCCESS + + """, + job="demo", + branch="master", + number=15, + ) + + assert evidence.revision == "abc123" + assert evidence.result is None + assert evidence.building is True + + +def test_jenkins_evidence_reads_only_top_level_terminal_result(): + evidence_reader = _load("jenkins_build_evidence") + evidence = evidence_reader.parse_build_xml( + """ + + abc123 + 1000250 + FAILURE + SUCCESS + + """, + job="demo", + branch=None, + number=4, + ) + + assert evidence.result == "FAILURE" + assert evidence.building is False + assert evidence.duration_ms == 250 + + +def test_jenkins_evidence_removes_hidden_links_and_ansi_control_sequences(): + evidence_reader = _load("jenkins_build_evidence") + + value = ( + "before\n\x1b[8mha:////private-jenkins-payload\x1b[0m[Pipeline] sh\n" + "\x1b[31m5 passed\x1b[0m\n" + ) + + assert evidence_reader.sanitize_log(value) == "before\n[Pipeline] sh\n5 passed\n" + + +def test_jenkins_evidence_falls_back_to_multibranch_suffix(monkeypatch): + evidence_reader = _load("jenkins_build_evidence") + calls = [] + + def fake_find(job, branch, commit, log_lines): + calls.append((job, branch, commit, log_lines)) + if job == "demo": + raise ValueError("not a multibranch job") + return "evidence" + + monkeypatch.setattr(evidence_reader, "_find_build_for_job", fake_find) + + assert evidence_reader.find_build("demo", "master", "abc", 20) == "evidence" + assert [call[0] for call in calls] == ["demo", "demo-branches"] + + +@pytest.mark.parametrize("branch", ["../master", "/master", "feature//unsafe"]) +def test_jenkins_evidence_rejects_unsafe_branch_paths(branch: str): + evidence_reader = _load("jenkins_build_evidence") + + with pytest.raises(ValueError): + evidence_reader._validate_branch(branch) diff --git a/testing/tests/test_hermes_runtime_stage_coverage.py b/testing/tests/test_hermes_runtime_stage_coverage.py index 3ca8e496..cae8066e 100644 --- a/testing/tests/test_hermes_runtime_stage_coverage.py +++ b/testing/tests/test_hermes_runtime_stage_coverage.py @@ -22,6 +22,7 @@ def _stage(tmp_path: Path, monkeypatch): monkeypatch.setattr(module, "RUNTIME_ROOT", runtime) monkeypatch.setattr(module, "PERSISTENT_HOME", home) monkeypatch.setattr(module.os, "chown", lambda *_args: None) + monkeypatch.setattr(module.os, "fchown", lambda *_args: None) return module, vault, runtime, home diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py index 8f1b2ca9..2a0fad44 100644 --- a/testing/tests/test_hermes_scm_broker.py +++ b/testing/tests/test_hermes_scm_broker.py @@ -48,7 +48,7 @@ def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): api.authorize_request("GET", path, None) -def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces(): +def test_receive_pack_allows_only_new_namespaced_feature_branch(): broker = _load("scm_broker") zero = b"0" * 40 commit = b"1" * 40 @@ -57,18 +57,9 @@ def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces(): _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), "runtime-sentinel", ) - broker._validate_receive_pack( - _receive_command(zero, commit, b"refs/heads/wt/t_deadbeef"), - "runtime-sentinel", - ) - broker._validate_receive_pack( - _receive_command(zero, commit, b"refs/heads/review/t_deadbeef"), - "runtime-sentinel", - ) for old, new, ref in ( (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), (commit, zero, b"refs/heads/hermes/focused-fix"), - (commit, commit, b"refs/heads/hermes/focused-fix"), (zero, commit, b"refs/heads/main"), (zero, commit, b"refs/heads/master"), (zero, commit, b"refs/tags/release"),