atlas-iac/scripts/ops/hermes_suite_probe.py

#!/usr/bin/env python3
"""Run only server-provided synthetic suites through the authenticated LAN API.

Set SUITE_PLANNING_TOKEN privately. No provider or cluster credentials are used.
The report contains operational metadata and synthetic family outputs only.
"""
import argparse
from collections import Counter
import http.client
import json
import os
from pathlib import Path
import socket
import ssl
import time
from urllib.error import HTTPError
from urllib.request import HTTPSHandler, HTTPRedirectHandler, ProxyHandler, Request, build_opener
import uuid

HOST = "worker.bstein.dev"
ADDRESS = "192.168.22.50"
BASE = "https://worker.bstein.dev/suite-planning"


class Connection(http.client.HTTPSConnection):
    """Connect directly to the LAN IP while verifying TLS against the hostname."""

    def connect(self):
        sock = socket.create_connection((ADDRESS, 443), self.timeout)
        self.sock = self._context.wrap_socket(sock, server_hostname=HOST)


class TLS(HTTPSHandler):
    """Keep hostname validation and the operating system's trusted CA store."""

    def https_open(self, req):
        return self.do_open(Connection, req, context=ssl.create_default_context())


class NoRedirect(HTTPRedirectHandler):
    """Redirects are errors, never alternate destinations for credentials."""

    def redirect_request(self, *args, **kwargs):
        return None


def api(path, value=None, *, key=None, token=None, method=None):
    """Perform one bounded HTTPS request with no proxy, redirects, or retry."""
    headers = {"Authorization": "Bearer " + (token if token is not None else os.environ["SUITE_PLANNING_TOKEN"])}
    if key:
        headers["Idempotency-Key"] = key
    body = None
    if value is not None:
        body = json.dumps(value, separators=(",", ":")).encode()
        headers["Content-Type"] = "application/json"
    request = Request(BASE + path, data=body, headers=headers, method=method)
    opener = build_opener(ProxyHandler({}), NoRedirect(), TLS())
    try:
        with opener.open(request, timeout=45) as response:
            return response.status, json.load(response)
    except HTTPError as exc:
        try:
            try:
                value = json.load(exc)
            except ValueError:
                value = {"error": {"code": "request_too_large" if exc.code == 413 else "http_error"}}
            return exc.code, value
        finally:
            exc.close()


def run(size):
    """Submit a complete fixture, verify idempotency, poll, and audit aliases."""
    code, request = api(f"/v1/synthetic/{size}")
    if code != 200:
        raise RuntimeError("fixture retrieval failed")
    request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
    request["execution"] = {"strategy": "whole_suite", "max_seconds": 900, "max_cost_usd": 5}
    code, preflight = api("/v1/preflight", request)
    if code != 200:
        return {"size": size, "status": code, "preflight": preflight}
    key = str(uuid.uuid4())
    started = time.monotonic()
    code, job = api("/v1/jobs", request, key=key)
    if code != 202:
        return {"size": size, "status": code, "submission": job}
    repeated_status, repeated = api("/v1/jobs", request, key=key)
    assert repeated_status == 200 and repeated["job_id"] == job["job_id"]
    job_id = job["job_id"]
    print(json.dumps({"size": size, "job_id": job_id, "status": "accepted"}), flush=True)
    while time.monotonic() - started < 960:
        code, result = api(f"/v1/jobs/{job_id}/result")
        if code != 200 or result["status"] in {"completed", "failed", "cancelled"}:
            break
        time.sleep(2)
    else:
        raise RuntimeError("poll deadline exceeded; retain job ID and idempotency key")
    coverage = None
    if "result" in result:
        coverage = Counter(c["alias"] for c in request["cases"]) == Counter(
            a for g in result["result"]["groups"] for a in g["members"])
    return {"size": size, "http_status": code,
            "request_bytes": len(json.dumps(request, separators=(",", ":")).encode()),
            "client_wall_seconds": round(time.monotonic() - started, 3),
            "exact_alias_coverage": coverage, "job": result}


def main():
    """Write an optional synthetic-only report and print metadata summaries."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--sizes", nargs="+", type=int, choices=(14, 75, 363), default=[14, 75, 363])
    parser.add_argument("--report", type=Path)
    args = parser.parse_args()
    assert api("/healthz", token="invalid")[0] == 401
    assert api("/healthz")[0] == 200
    assert api("/api/pull")[0] == 404
    reports = []
    for size in args.sizes:
        report = run(size)
        reports.append(report)
        print(json.dumps({k: v for k, v in report.items() if k != "job"}), flush=True)
        if args.report:
            args.report.write_text(json.dumps(reports, indent=2) + "\n")
    if any(report.get("exact_alias_coverage") is not True for report in reports):
        raise SystemExit(1)


if __name__ == "__main__":
    main()