hermes: verify native planning runtime before readiness

scripts/ops/hermes_suite_probe.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Run only server-provided synthetic suites through the authenticated LAN API.
+
+Set SUITE_PLANNING_TOKEN privately. No provider or cluster credentials are used.
+The report contains operational metadata and synthetic family outputs only.
+"""
+import argparse
+from collections import Counter
+import http.client
+import json
+import os
+from pathlib import Path
+import socket
+import ssl
+import time
+from urllib.error import HTTPError
+from urllib.request import HTTPSHandler, HTTPRedirectHandler, ProxyHandler, Request, build_opener
+import uuid
+
+HOST = "worker.bstein.dev"
+ADDRESS = "192.168.22.50"
+BASE = "https://worker.bstein.dev/suite-planning"
+
+
+class Connection(http.client.HTTPSConnection):
+    """Connect directly to the LAN IP while verifying TLS against the hostname."""
+
+    def connect(self):
+        sock = socket.create_connection((ADDRESS, 443), self.timeout)
+        self.sock = self._context.wrap_socket(sock, server_hostname=HOST)
+
+
+class TLS(HTTPSHandler):
+    """Keep hostname validation and the operating system's trusted CA store."""
+
+    def https_open(self, req):
+        return self.do_open(Connection, req, context=ssl.create_default_context())
+
+
+class NoRedirect(HTTPRedirectHandler):
+    """Redirects are errors, never alternate destinations for credentials."""
+
+    def redirect_request(self, *args, **kwargs):
+        return None
+
+
+def api(path, value=None, *, key=None, token=None, method=None):
+    """Perform one bounded HTTPS request with no proxy, redirects, or retry."""
+    headers = {"Authorization": "Bearer " + (token if token is not None else os.environ["SUITE_PLANNING_TOKEN"])}
+    if key:
+        headers["Idempotency-Key"] = key
+    body = None
+    if value is not None:
+        body = json.dumps(value, separators=(",", ":")).encode()
+        headers["Content-Type"] = "application/json"
+    request = Request(BASE + path, data=body, headers=headers, method=method)
+    opener = build_opener(ProxyHandler({}), NoRedirect(), TLS())
+    try:
+        with opener.open(request, timeout=45) as response:
+            return response.status, json.load(response)
+    except HTTPError as exc:
+        try:
+            return exc.code, json.load(exc)
+        finally:
+            exc.close()
+
+
+def run(size):
+    """Submit a complete fixture, verify idempotency, poll, and audit aliases."""
+    code, request = api(f"/v1/synthetic/{size}")
+    if code != 200:
+        raise RuntimeError("fixture retrieval failed")
+    request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
+    request["execution"] = {"strategy": "whole_suite", "max_seconds": 900, "max_cost_usd": 5}
+    code, preflight = api("/v1/preflight", request)
+    if code != 200:
+        return {"size": size, "status": code, "preflight": preflight}
+    key = str(uuid.uuid4())
+    started = time.monotonic()
+    code, job = api("/v1/jobs", request, key=key)
+    if code != 202:
+        return {"size": size, "status": code, "submission": job}
+    repeated_status, repeated = api("/v1/jobs", request, key=key)
+    assert repeated_status == 200 and repeated["job_id"] == job["job_id"]
+    job_id = job["job_id"]
+    print(json.dumps({"size": size, "job_id": job_id, "status": "accepted"}), flush=True)
+    while time.monotonic() - started < 960:
+        code, result = api(f"/v1/jobs/{job_id}/result")
+        if code != 200 or result["status"] in {"completed", "failed", "cancelled"}:
+            break
+        time.sleep(2)
+    else:
+        raise RuntimeError("poll deadline exceeded; retain job ID and idempotency key")
+    coverage = None
+    if "result" in result:
+        coverage = Counter(c["alias"] for c in request["cases"]) == Counter(
+            a for g in result["result"]["groups"] for a in g["members"])
+    return {"size": size, "http_status": code, "request_bytes": len(json.dumps(request).encode()),
+            "client_wall_seconds": round(time.monotonic() - started, 3),
+            "exact_alias_coverage": coverage, "job": result}
+
+
+def main():
+    """Write an optional synthetic-only report and print metadata summaries."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--sizes", nargs="+", type=int, choices=(14, 75, 363), default=[14, 75, 363])
+    parser.add_argument("--report", type=Path)
+    args = parser.parse_args()
+    assert api("/healthz", token="invalid")[0] == 401
+    assert api("/healthz")[0] == 200
+    assert api("/api/pull")[0] == 404
+    reports = []
+    for size in args.sizes:
+        report = run(size)
+        reports.append(report)
+        print(json.dumps({k: v for k, v in report.items() if k != "job"}), flush=True)
+        if args.report:
+            args.report.write_text(json.dumps(reports, indent=2) + "\n")
+    if any(report.get("exact_alias_coverage") is not True for report in reports):
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

services/hermes/scripts/suite_api.py

@@ -9,6 +9,7 @@ import json
 import os
 from pathlib import Path
 import re
+import subprocess
 import threading
 
 from suite_contract import (MAX_BODY, MAX_CASES, MAX_RESULT, MODELS, REVISION,
@@ -186,6 +187,10 @@ def main():
     binary_hash = hashlib.sha256(Path("/opt/cli/claude").read_bytes()).hexdigest()
     if binary_hash != os.environ["PLANNING_CLAUDE_SHA256"]:
         raise SystemExit("Pinned CLI binary mismatch")
+    version = subprocess.run(["/opt/cli/claude", "--version"], capture_output=True,
+                             text=True, timeout=10, check=True)
+    if version.stdout.strip() != "2.1.226 (Claude Code)":
+        raise SystemExit("Pinned CLI version mismatch")
     private = ThreadingHTTPServer(("0.0.0.0", 9001), Decision)
     threading.Thread(target=private.serve_forever, daemon=True).start()
     server = ThreadingHTTPServer(("0.0.0.0", 9000), Handler)

services/hermes/suite-planner-deployment.yaml

@@ -36,7 +36,7 @@ spec:
         app: hermes-suite-planner
       annotations:
         fluentbit.io/exclude: "true"
-        ai.bstein.dev/config-rev: suite-v1-20260929
+        ai.bstein.dev/config-rev: suite-v2-20260929
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/agent-pre-populate-only: "true"
         vault.hashicorp.com/agent-init-first: "true"
@@ -87,7 +87,7 @@ spec:
           type: RuntimeDefault
       initContainers:
         - name: stage-cli
-          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
+          image: python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6
           command: [python, -c]
           args:
             - |
@@ -109,7 +109,7 @@ spec:
             limits: {cpu: "1", memory: 1Gi}
       containers:
         - name: planner
-          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
+          image: python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6
           command: [python, /opt/planner/suite_api.py]
           env:
             - {name: PYTHONDONTWRITEBYTECODE, value: "1"}