From d7d2d5430b47e0dc5733ba642b4b501b50ba2fbd Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 29 Sep 2026 07:56:36 -0500 Subject: [PATCH] hermes: verify native planning runtime before readiness --- scripts/ops/hermes_suite_probe.py | 124 ++++++++++++++++++ services/hermes/scripts/suite_api.py | 5 + services/hermes/suite-planner-deployment.yaml | 6 +- 3 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 scripts/ops/hermes_suite_probe.py diff --git a/scripts/ops/hermes_suite_probe.py b/scripts/ops/hermes_suite_probe.py new file mode 100644 index 00000000..4b136fac --- /dev/null +++ b/scripts/ops/hermes_suite_probe.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Run only server-provided synthetic suites through the authenticated LAN API. + +Set SUITE_PLANNING_TOKEN privately. No provider or cluster credentials are used. +The report contains operational metadata and synthetic family outputs only. +""" +import argparse +from collections import Counter +import http.client +import json +import os +from pathlib import Path +import socket +import ssl +import time +from urllib.error import HTTPError +from urllib.request import HTTPSHandler, HTTPRedirectHandler, ProxyHandler, Request, build_opener +import uuid + +HOST = "worker.bstein.dev" +ADDRESS = "192.168.22.50" +BASE = "https://worker.bstein.dev/suite-planning" + + +class Connection(http.client.HTTPSConnection): + """Connect directly to the LAN IP while verifying TLS against the hostname.""" + + def connect(self): + sock = socket.create_connection((ADDRESS, 443), self.timeout) + self.sock = self._context.wrap_socket(sock, server_hostname=HOST) + + +class TLS(HTTPSHandler): + """Keep hostname validation and the operating system's trusted CA store.""" + + def https_open(self, req): + return self.do_open(Connection, req, context=ssl.create_default_context()) + + +class NoRedirect(HTTPRedirectHandler): + """Redirects are errors, never alternate destinations for credentials.""" + + def redirect_request(self, *args, **kwargs): + return None + + +def api(path, value=None, *, key=None, token=None, method=None): + """Perform one bounded HTTPS request with no proxy, redirects, or retry.""" + headers = {"Authorization": "Bearer " + (token if token is not None else os.environ["SUITE_PLANNING_TOKEN"])} + if key: + headers["Idempotency-Key"] = key + body = None + if value is not None: + body = json.dumps(value, separators=(",", ":")).encode() + headers["Content-Type"] = "application/json" + request = Request(BASE + path, data=body, headers=headers, method=method) + opener = build_opener(ProxyHandler({}), NoRedirect(), TLS()) + try: + with opener.open(request, timeout=45) as response: + return response.status, json.load(response) + except HTTPError as exc: + try: + return exc.code, json.load(exc) + finally: + exc.close() + + +def run(size): + """Submit a complete fixture, verify idempotency, poll, and audit aliases.""" + code, request = api(f"/v1/synthetic/{size}") + if code != 200: + raise RuntimeError("fixture retrieval failed") + request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]} + request["execution"] = {"strategy": "whole_suite", "max_seconds": 900, "max_cost_usd": 5} + code, preflight = api("/v1/preflight", request) + if code != 200: + return {"size": size, "status": code, "preflight": preflight} + key = str(uuid.uuid4()) + started = time.monotonic() + code, job = api("/v1/jobs", request, key=key) + if code != 202: + return {"size": size, "status": code, "submission": job} + repeated_status, repeated = api("/v1/jobs", request, key=key) + assert repeated_status == 200 and repeated["job_id"] == job["job_id"] + job_id = job["job_id"] + print(json.dumps({"size": size, "job_id": job_id, "status": "accepted"}), flush=True) + while time.monotonic() - started < 960: + code, result = api(f"/v1/jobs/{job_id}/result") + if code != 200 or result["status"] in {"completed", "failed", "cancelled"}: + break + time.sleep(2) + else: + raise RuntimeError("poll deadline exceeded; retain job ID and idempotency key") + coverage = None + if "result" in result: + coverage = Counter(c["alias"] for c in request["cases"]) == Counter( + a for g in result["result"]["groups"] for a in g["members"]) + return {"size": size, "http_status": code, "request_bytes": len(json.dumps(request).encode()), + "client_wall_seconds": round(time.monotonic() - started, 3), + "exact_alias_coverage": coverage, "job": result} + + +def main(): + """Write an optional synthetic-only report and print metadata summaries.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", nargs="+", type=int, choices=(14, 75, 363), default=[14, 75, 363]) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + assert api("/healthz", token="invalid")[0] == 401 + assert api("/healthz")[0] == 200 + assert api("/api/pull")[0] == 404 + reports = [] + for size in args.sizes: + report = run(size) + reports.append(report) + print(json.dumps({k: v for k, v in report.items() if k != "job"}), flush=True) + if args.report: + args.report.write_text(json.dumps(reports, indent=2) + "\n") + if any(report.get("exact_alias_coverage") is not True for report in reports): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/services/hermes/scripts/suite_api.py b/services/hermes/scripts/suite_api.py index bf26f5a1..92acee69 100644 --- a/services/hermes/scripts/suite_api.py +++ b/services/hermes/scripts/suite_api.py @@ -9,6 +9,7 @@ import json import os from pathlib import Path import re +import subprocess import threading from suite_contract import (MAX_BODY, MAX_CASES, MAX_RESULT, MODELS, REVISION, @@ -186,6 +187,10 @@ def main(): binary_hash = hashlib.sha256(Path("/opt/cli/claude").read_bytes()).hexdigest() if binary_hash != os.environ["PLANNING_CLAUDE_SHA256"]: raise SystemExit("Pinned CLI binary mismatch") + version = subprocess.run(["/opt/cli/claude", "--version"], capture_output=True, + text=True, timeout=10, check=True) + if version.stdout.strip() != "2.1.226 (Claude Code)": + raise SystemExit("Pinned CLI version mismatch") private = ThreadingHTTPServer(("0.0.0.0", 9001), Decision) threading.Thread(target=private.serve_forever, daemon=True).start() server = ThreadingHTTPServer(("0.0.0.0", 9000), Handler) diff --git a/services/hermes/suite-planner-deployment.yaml b/services/hermes/suite-planner-deployment.yaml index e22296e7..70513be2 100644 --- a/services/hermes/suite-planner-deployment.yaml +++ b/services/hermes/suite-planner-deployment.yaml @@ -36,7 +36,7 @@ spec: app: hermes-suite-planner annotations: fluentbit.io/exclude: "true" - ai.bstein.dev/config-rev: suite-v1-20260929 + ai.bstein.dev/config-rev: suite-v2-20260929 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" @@ -87,7 +87,7 @@ spec: type: RuntimeDefault initContainers: - name: stage-cli - image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + image: python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6 command: [python, -c] args: - | @@ -109,7 +109,7 @@ spec: limits: {cpu: "1", memory: 1Gi} containers: - name: planner - image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + image: python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6 command: [python, /opt/planner/suite_api.py] env: - {name: PYTHONDONTWRITEBYTECODE, value: "1"}