80 lines
4.1 KiB
Python
Executable File
80 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run only the fixed 363-case capacity fixture through one async suite job.
|
|
|
|
Server-side acceptance uses an isolated metadata DB and the deployed pinned CLI.
|
|
No real job/source input is accepted. Output is operational metadata and scores.
|
|
"""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
sys.path.insert(0, os.environ.get("SUITE_PROBE_MODULE_DIR", "/opt/planner"))
|
|
from suite_capacity_fixture import fixture
|
|
from suite_contract import encoded, preflight, validate_request, validate_result
|
|
from suite_jobs import Jobs
|
|
from suite_sizing import balanced_sizes
|
|
from suite_synthetic import score
|
|
|
|
|
|
def field_sizes(cases):
|
|
"""Record only byte-length distribution, never source field values."""
|
|
result = {}
|
|
for field in ("description", "preconditions", "success_criteria", "case_type"):
|
|
sizes = sorted(len(c[field].encode()) for c in cases)
|
|
result[field] = {"min": min(sizes), "median": sizes[len(sizes)//2],
|
|
"p95": sizes[int((len(sizes)-1)*.95)], "max": max(sizes),
|
|
"mean": round(sum(sizes)/len(sizes), 2)}
|
|
return result
|
|
|
|
|
|
def main():
|
|
"""Require all five logical stages; retain no generated descriptions in report."""
|
|
raw, expected = fixture()
|
|
source_bytes = len(encoded(raw))
|
|
raw["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
|
|
raw["execution"] = {"strategy": "whole_suite", "max_seconds": 7200}
|
|
request = validate_request(raw, ["claude"])
|
|
selected = preflight(request)
|
|
report = {"test": "synthetic_capacity_363", "source_bytes": source_bytes,
|
|
"request_bytes": len(encoded(request)), "case_count": len(request["cases"]),
|
|
"field_bytes": field_sizes(request["cases"]), "preflight": selected,
|
|
"real_field_distribution_known": False, "automatic_job_retries": 0}
|
|
with tempfile.TemporaryDirectory(prefix="suite-capacity-probe-", dir="/jobs") as directory:
|
|
jobs = Jobs(Path(directory) / "jobs.sqlite")
|
|
job, _ = jobs.submit("synthetic-capacity-probe", "synthetic-capacity-363", request,
|
|
selected, "192.168.22.8")
|
|
last_report = 0.0
|
|
while job["status"] in {"accepted", "running", "cancelling"}:
|
|
if time.monotonic() - last_report >= 20:
|
|
p = job.get("execution_progress", {})
|
|
print(json.dumps({k: p.get(k) for k in (
|
|
"current_pass", "review_batch", "completed_model_passes", "job_elapsed_seconds",
|
|
"job_remaining_seconds", "cost_used_usd_estimate")}), file=sys.stderr, flush=True)
|
|
last_report = time.monotonic()
|
|
time.sleep(2)
|
|
job = jobs.get(job["job_id"], "synthetic-capacity-probe")
|
|
report.update(status=job["status"], metadata=job)
|
|
if job["status"] == "completed":
|
|
result = jobs.get(job["job_id"], "synthetic-capacity-probe", result=True)
|
|
validate_result(result["result"], request)
|
|
stages = {p["stage"] for p in job["passes"]}
|
|
assert {"large_family_review", "decision_audit"} <= stages
|
|
assert ({"proposal_a", "proposal_b", "reconciliation"} <= stages or
|
|
{"implementation_profiles", "profile_proposal_a", "profile_proposal_b",
|
|
"profile_reconciliation", "source_review"} <= stages)
|
|
review = result["review_summary"]
|
|
assert all(d["part_sizes"] == balanced_sizes(d["natural_case_count"]) for d in review["capacity_divisions"])
|
|
report.update(response_bytes=len(encoded(result)), exact_coverage=True,
|
|
max_group_size=max(len(g["members"]) for g in result["result"]["groups"]),
|
|
names_valid=True, balanced_packaging=True, counts=review["counts"],
|
|
natural_quality=score({"groups": review["natural_families"]}, expected))
|
|
print(json.dumps(report, sort_keys=True), flush=True)
|
|
return 0 if report["status"] == "completed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|