111 lines
5.8 KiB
Python
Executable File
111 lines
5.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Benchmark one synthetic async job in an isolated loopback API before rollout.
|
|
|
|
Uses the installed pinned CLI/account and scoped synthetic credential. Accepts
|
|
only built-in synthetic fixtures, never an input file or real job identifier.
|
|
HTTP requests retain the ordinary 45-second timeout. No service configuration
|
|
or live job database is modified by this probe.
|
|
"""
|
|
import argparse
|
|
from http.server import ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from urllib.error import HTTPError
|
|
from urllib.request import ProxyHandler, Request, build_opener
|
|
|
|
sys.path.insert(0, os.environ.get("SUITE_PROBE_MODULE_DIR", "/opt/planner"))
|
|
from suite_api import Handler
|
|
from suite_contract import EXECUTION_REVISION, PROMPT_REVISION, REVISION, encoded, validate_result
|
|
from suite_jobs import Jobs
|
|
from suite_synthetic import fixture, score
|
|
|
|
|
|
def main():
|
|
"""Submit once, poll safely, and report measured metadata and independent scores."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--size", type=int, choices=(14, 75, 363), default=75)
|
|
parser.add_argument("--max-seconds", type=int, default=7200)
|
|
args = parser.parse_args()
|
|
request, expected = fixture(args.size)
|
|
request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
|
|
request["execution"] = {"strategy": "whole_suite", "max_seconds": args.max_seconds, "max_cost_usd": None}
|
|
token = Path("/vault/secrets/synthetic-token").read_text().strip()
|
|
opener = build_opener(ProxyHandler({}))
|
|
with tempfile.TemporaryDirectory(prefix="suite-budget-probe-", dir="/jobs") as directory:
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
server.jobs = Jobs(Path(directory) / "jobs.sqlite")
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
|
|
def http(path, value=None, key=None):
|
|
headers = {"Authorization": "Bearer " + token, "X-Forwarded-For": "192.168.22.8",
|
|
"Content-Type": "application/json"}
|
|
if key:
|
|
headers["Idempotency-Key"] = key
|
|
call = Request("http://127.0.0.1:" + str(server.server_port) + path,
|
|
data=encoded(value) if value is not None else None, headers=headers)
|
|
try:
|
|
with opener.open(call, timeout=45) as response:
|
|
return response.status, json.load(response)
|
|
except HTTPError as exc:
|
|
with exc:
|
|
return exc.code, json.load(exc)
|
|
|
|
started, last_report = time.monotonic(), 0.0
|
|
report = {"test": "isolated_async_synthetic_budget_benchmark", "case_count": args.size,
|
|
"request_bytes": len(encoded(request)), "configuration_revision": REVISION,
|
|
"execution_revision": EXECUTION_REVISION, "prompt_revision": PROMPT_REVISION,
|
|
"http_timeout_seconds": 45, "automatic_job_retries": 0}
|
|
try:
|
|
status, capabilities = http("/v1/capabilities")
|
|
assert status == 200 and capabilities["max_seconds"] == 7200
|
|
assert capabilities["default_max_seconds"] == 1800
|
|
invalid = {**request, "execution": {**request["execution"], "max_seconds": 7201}}
|
|
status, rejected = http("/v1/preflight", invalid)
|
|
assert status == 400 and rejected["error"]["code"] == "invalid_timeout"
|
|
status, checked = http("/v1/preflight", request)
|
|
assert status == 200 and checked["execution"]["max_seconds"] == args.max_seconds
|
|
report.update(capabilities_limits={k: capabilities[k] for k in (
|
|
"max_seconds", "default_max_seconds", "max_cost_usd")}, preflight=checked)
|
|
status, job = http("/v1/jobs", request, "synthetic-long-budget-probe")
|
|
assert status == 202 and job["execution"]["max_seconds"] == args.max_seconds
|
|
path = "/v1/jobs/" + job["job_id"]
|
|
replay_status, replay = http("/v1/jobs", request, "synthetic-long-budget-probe")
|
|
assert replay_status == 200 and replay["job_id"] == job["job_id"]
|
|
report["idempotent_replay_verified"] = True
|
|
while job["status"] in {"accepted", "running", "cancelling"}:
|
|
if time.monotonic() - last_report >= 20:
|
|
progress = job.get("execution_progress", {})
|
|
print(json.dumps({k: progress.get(k) for k in (
|
|
"current_pass", "completed_model_passes", "job_elapsed_seconds",
|
|
"job_remaining_seconds", "cost_used_usd_estimate")}), file=sys.stderr, flush=True)
|
|
last_report = time.monotonic()
|
|
time.sleep(2)
|
|
_, job = http(path)
|
|
report["status"] = job["status"]
|
|
report["job_metadata"] = job
|
|
if job["status"] == "completed":
|
|
status, result = http(path + "/result")
|
|
assert status == 200
|
|
validate_result(result["result"], request)
|
|
natural = {"groups": result["review_summary"]["natural_families"]}
|
|
report.update(natural_quality=score(natural, expected),
|
|
counts=result["review_summary"]["counts"], exact_final_coverage=True)
|
|
allocations = [p["allocated_seconds"] for p in job["passes"]]
|
|
assert allocations[0] > 1800
|
|
assert all(a > b for a, b in zip(allocations, allocations[1:]))
|
|
assert job["cli_diagnostics"]["subprocess_timeout_seconds"] == allocations[-1]
|
|
report["wall_seconds"] = round(time.monotonic() - started, 3)
|
|
print(json.dumps(report, sort_keys=True), flush=True)
|
|
return 0 if report["status"] == "completed" else 1
|
|
finally:
|
|
server.shutdown()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|