68 lines
3.2 KiB
Python
Executable File
68 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Compare direct and hierarchical execution on a fixed synthetic 14-case suite.
|
|
|
|
Run inside the planner with staged modules. No real source or job ID is accepted.
|
|
Reports operational metadata plus synthetic-only quality evidence. No live job DB.
|
|
"""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
sys.path.insert(0, os.environ.get('SUITE_PROBE_MODULE_DIR', '/opt/planner'))
|
|
import suite_profiles
|
|
from suite_contract import preflight, validate_request, validate_result
|
|
from suite_multipass import generate
|
|
from suite_synthetic import PATTERNS, score
|
|
|
|
|
|
def main():
|
|
"""Use interleaved mechanisms, identical records, and cross-processing-batch peers."""
|
|
cases, expected = [], {}
|
|
for i in range(14):
|
|
family, action, assertion, setup = PATTERNS[i % 2]
|
|
alias = 'CASE-REC-%02d' % i
|
|
cases.append({'alias': alias, 'description': action,
|
|
'success_criteria': assertion, 'preconditions': setup,
|
|
'case_type': 'nominal' if i % 3 else 'fault injection'})
|
|
expected[alias] = family
|
|
cases[-2] = {**cases[0], 'alias': cases[-2]['alias']}
|
|
request = validate_request({'campaign':'SYNTHETIC', 'suite':'RECOVERY-14', 'cases':cases,
|
|
'routing':{'allow_external':True,'allowed_external_providers':['claude']},
|
|
'execution':{'strategy':'whole_suite','max_seconds':900}}, ['claude'])
|
|
suite_profiles.BATCH_CASES = 4 # Test processing-boundary independence explicitly.
|
|
for mode in ('direct','hierarchical'):
|
|
last = [0]
|
|
def progress(p):
|
|
if time.monotonic()-last[0] >= 20:
|
|
print(json.dumps({'mode':mode, **{k:p.get(k) for k in (
|
|
'current_pass','completed_model_passes','job_elapsed_seconds','retry_count')}}),
|
|
file=sys.stderr, flush=True)
|
|
last[0] = time.monotonic()
|
|
started = time.monotonic()
|
|
try:
|
|
result, metadata = generate(request, {**preflight(request),'execution_mode':mode},
|
|
threading.Event(), '192.168.22.8', progress,
|
|
credential_scope={'owner':'synthetic-recovery-probe','providers':['claude']})
|
|
validate_result(result, request)
|
|
quality = score({'groups':metadata['review_summary']['natural_families']}, expected)
|
|
print(json.dumps({'mode':mode,'status':'completed','seconds':time.monotonic()-started,
|
|
'quality':quality,'usage':metadata['usage'],'cost_usd_estimate':metadata['cost_usd_estimate'],
|
|
'passes':[{k:r.get(k) for k in ('stage','wall_seconds','cli_turns','max_turns','input_bytes')}
|
|
for r in metadata['passes']],
|
|
'natural_groups':metadata['review_summary']['natural_families'],
|
|
'final_groups':result['groups']}, sort_keys=True),flush=True)
|
|
except Exception as exc:
|
|
if hasattr(exc,'document'):
|
|
print(json.dumps({'mode':mode,'status':'failed',**exc.document()}),flush=True)
|
|
else:
|
|
print(json.dumps({'mode':mode,'status':'failed','error_type':type(exc).__name__}),flush=True)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|