66 lines
3.1 KiB
Python
66 lines
3.1 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Run exactly one synthetic 61-case proposal through the deployed Claude adapter.
|
||
|
|
|
||
|
|
Server-side only. Accepts no case input, never resumes a suite, and prints only
|
||
|
|
operational metadata and synthetic coverage scores. This is not a full workflow.
|
||
|
|
"""
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
|
||
|
|
sys.path.insert(0, "/opt/planner")
|
||
|
|
from suite_contract import (EXECUTION_REVISION, PROMPT_REVISION, Problem,
|
||
|
|
encoded, validate_partition, validate_request)
|
||
|
|
from suite_multipass import Workflow, ordered_request, preflight_workflow
|
||
|
|
from suite_synthetic import fixture, score
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""Exercise the failing stage once, with complete synthetic fields and aliases."""
|
||
|
|
request, expected = fixture(75)
|
||
|
|
request["cases"] = request["cases"][:59] + request["cases"][-2:]
|
||
|
|
request["suite"] = "SYNTHETIC-PROPOSAL-61"
|
||
|
|
for case in request["cases"]:
|
||
|
|
case["success_criteria"] += (
|
||
|
|
" Record the configured input, observed result, and pass/fail outcome separately."
|
||
|
|
" Repeat the observation after fixture reset to distinguish persistent state from"
|
||
|
|
" the intended variation. Missing procedure details remain unknown; do not infer"
|
||
|
|
" additional interfaces or equipment from the shared reference placeholder.")
|
||
|
|
aliases = {case["alias"] for case in request["cases"]}
|
||
|
|
expected = {alias: family for alias, family in expected.items() if alias in aliases}
|
||
|
|
request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]}
|
||
|
|
request = validate_request(request, ["claude"])
|
||
|
|
selected = preflight_workflow(request)
|
||
|
|
report = {"test": "synthetic_proposal_a_only", "case_count": len(aliases),
|
||
|
|
"request_bytes": len(encoded(request)), "selection": selected,
|
||
|
|
"execution_revision": EXECUTION_REVISION, "prompt_revision": PROMPT_REVISION,
|
||
|
|
"automatic_retries": 0}
|
||
|
|
started, last_report = time.monotonic(), 0.0
|
||
|
|
|
||
|
|
def progress(value):
|
||
|
|
"""Publish a content-free heartbeat during the single model invocation."""
|
||
|
|
nonlocal last_report
|
||
|
|
if time.monotonic() - last_report >= 20:
|
||
|
|
print(json.dumps({key: value.get(key) for key in (
|
||
|
|
"current_pass", "completed_model_passes", "job_elapsed_seconds",
|
||
|
|
"cli_running", "cli_output_bytes")}), file=sys.stderr, flush=True)
|
||
|
|
last_report = time.monotonic()
|
||
|
|
|
||
|
|
workflow = Workflow(request, selected, threading.Event(), "192.168.22.8", progress)
|
||
|
|
try:
|
||
|
|
source = ordered_request(request)
|
||
|
|
result = workflow.call("proposal_a", source)
|
||
|
|
validate_partition(result, source, name_limit=56, unique_names=False)
|
||
|
|
report.update(status="completed", quality=score(result, expected),
|
||
|
|
passes=workflow.records, metadata=workflow.last_metadata)
|
||
|
|
except Problem as exc:
|
||
|
|
report.update(status="failed", **exc.document())
|
||
|
|
report["wall_seconds"] = round(time.monotonic() - started, 3)
|
||
|
|
print(json.dumps(report, sort_keys=True), flush=True)
|
||
|
|
return 0 if report["status"] == "completed" else 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|