129 lines
5.0 KiB
Python
129 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Run only server-provided synthetic suites through the authenticated LAN API.
|
|
|
|
Set SUITE_PLANNING_TOKEN privately. No provider or cluster credentials are used.
|
|
The report contains operational metadata and synthetic family outputs only.
|
|
"""
|
|
import argparse
|
|
from collections import Counter
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import ssl
|
|
import time
|
|
from urllib.error import HTTPError
|
|
from urllib.request import HTTPSHandler, HTTPRedirectHandler, ProxyHandler, Request, build_opener
|
|
import uuid
|
|
|
|
HOST = "worker.bstein.dev"
|
|
ADDRESS = "192.168.22.50"
|
|
BASE = "https://worker.bstein.dev/suite-planning"
|
|
|
|
|
|
class Connection(http.client.HTTPSConnection):
|
|
"""Connect directly to the LAN IP while verifying TLS against the hostname."""
|
|
|
|
def connect(self):
|
|
sock = socket.create_connection((ADDRESS, 443), self.timeout)
|
|
self.sock = self._context.wrap_socket(sock, server_hostname=HOST)
|
|
|
|
|
|
class TLS(HTTPSHandler):
|
|
"""Keep hostname validation and the operating system's trusted CA store."""
|
|
|
|
def https_open(self, req):
|
|
return self.do_open(Connection, req, context=ssl.create_default_context())
|
|
|
|
|
|
class NoRedirect(HTTPRedirectHandler):
|
|
"""Redirects are errors, never alternate destinations for credentials."""
|
|
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
def api(path, value=None, *, key=None, token=None, method=None):
|
|
"""Perform one bounded HTTPS request with no proxy, redirects, or retry."""
|
|
headers = {"Authorization": "Bearer " + (token if token is not None else os.environ["SUITE_PLANNING_TOKEN"])}
|
|
if key:
|
|
headers["Idempotency-Key"] = key
|
|
body = None
|
|
if value is not None:
|
|
body = json.dumps(value, separators=(",", ":")).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
request = Request(BASE + path, data=body, headers=headers, method=method)
|
|
opener = build_opener(ProxyHandler({}), NoRedirect(), TLS())
|
|
try:
|
|
with opener.open(request, timeout=45) as response:
|
|
return response.status, json.load(response)
|
|
except HTTPError as exc:
|
|
try:
|
|
try:
|
|
value = json.load(exc)
|
|
except ValueError:
|
|
value = {"error": {"code": "request_too_large" if exc.code == 413 else "http_error"}}
|
|
return exc.code, value
|
|
finally:
|
|
exc.close()
|
|
|
|
|
|
def run(size):
|
|
"""Submit a complete fixture, verify idempotency, poll, and audit aliases."""
|
|
code, request = api(f"/v1/synthetic/{size}")
|
|
if code != 200:
|
|
raise RuntimeError("fixture retrieval failed")
|
|
request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
|
|
request["execution"] = {"strategy": "whole_suite", "max_seconds": 900, "max_cost_usd": 5}
|
|
code, preflight = api("/v1/preflight", request)
|
|
if code != 200:
|
|
return {"size": size, "status": code, "preflight": preflight}
|
|
key = str(uuid.uuid4())
|
|
started = time.monotonic()
|
|
code, job = api("/v1/jobs", request, key=key)
|
|
if code != 202:
|
|
return {"size": size, "status": code, "submission": job}
|
|
repeated_status, repeated = api("/v1/jobs", request, key=key)
|
|
assert repeated_status == 200 and repeated["job_id"] == job["job_id"]
|
|
job_id = job["job_id"]
|
|
print(json.dumps({"size": size, "job_id": job_id, "status": "accepted"}), flush=True)
|
|
while time.monotonic() - started < 960:
|
|
code, result = api(f"/v1/jobs/{job_id}/result")
|
|
if code != 200 or result["status"] in {"completed", "failed", "cancelled"}:
|
|
break
|
|
time.sleep(2)
|
|
else:
|
|
raise RuntimeError("poll deadline exceeded; retain job ID and idempotency key")
|
|
coverage = None
|
|
if "result" in result:
|
|
coverage = Counter(c["alias"] for c in request["cases"]) == Counter(
|
|
a for g in result["result"]["groups"] for a in g["members"])
|
|
return {"size": size, "http_status": code, "request_bytes": len(json.dumps(request).encode()),
|
|
"client_wall_seconds": round(time.monotonic() - started, 3),
|
|
"exact_alias_coverage": coverage, "job": result}
|
|
|
|
|
|
def main():
|
|
"""Write an optional synthetic-only report and print metadata summaries."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--sizes", nargs="+", type=int, choices=(14, 75, 363), default=[14, 75, 363])
|
|
parser.add_argument("--report", type=Path)
|
|
args = parser.parse_args()
|
|
assert api("/healthz", token="invalid")[0] == 401
|
|
assert api("/healthz")[0] == 200
|
|
assert api("/api/pull")[0] == 404
|
|
reports = []
|
|
for size in args.sizes:
|
|
report = run(size)
|
|
reports.append(report)
|
|
print(json.dumps({k: v for k, v in report.items() if k != "job"}), flush=True)
|
|
if args.report:
|
|
args.report.write_text(json.dumps(reports, indent=2) + "\n")
|
|
if any(report.get("exact_alias_coverage") is not True for report in reports):
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|