76 lines
4.8 KiB
Python
Executable File
76 lines
4.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Synthetic native CLI turn-limit probe; all model HTTP goes to loopback.
|
|
|
|
Run inside the planner with: kubectl exec -i ... -- python < this-file
|
|
No hosted inference or live account credential is used.
|
|
"""
|
|
import json, threading, tempfile, pathlib, subprocess, sys, time
|
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
sys.path.insert(0, '/opt/planner')
|
|
from suite_backends import claude_environment, claude_command
|
|
from suite_contract import SYSTEM, prompt, validate_partition
|
|
from suite_policy import DISCOVERY_SCHEMA
|
|
from suite_synthetic import fixture
|
|
seen = []
|
|
request = fixture(14)[0]
|
|
source = prompt(request)
|
|
def strings(value):
|
|
if isinstance(value, str): yield value
|
|
elif isinstance(value, list):
|
|
for item in value: yield from strings(item)
|
|
elif isinstance(value, dict):
|
|
for item in value.values(): yield from strings(item)
|
|
class Mock(BaseHTTPRequestHandler):
|
|
def log_message(self, *args): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
seen.append({'stream': body.get('stream'), 'max_tokens': body.get('max_tokens'),
|
|
'http_timeout_header': self.headers.get('x-stainless-timeout'),
|
|
'whole_input': source in list(strings(body.get('messages', []))),
|
|
'system_preserved': any(SYSTEM in s for s in strings(body.get('system', [])))})
|
|
value = {} if len(seen) <= 3 else {'groups': [{'name': 'Synthetic', 'description': 'Mock only', 'members': [c['alias'] for c in request['cases']]}]}
|
|
block = {'type': 'tool_use', 'id': 'mock-' + str(len(seen)), 'name': 'StructuredOutput', 'input': value}
|
|
message = {'id': 'mock', 'type': 'message', 'role': 'assistant', 'model': 'claude-opus-4-8',
|
|
'content': [block], 'stop_reason': 'tool_use', 'stop_sequence': None,
|
|
'usage': {'input_tokens': 10, 'output_tokens': 5}}
|
|
if body.get('stream'):
|
|
initial = {**message, 'content': [], 'stop_reason': None, 'usage': {'input_tokens': 10, 'output_tokens': 0}}
|
|
events = [('message_start', {'type': 'message_start', 'message': initial}),
|
|
('content_block_start', {'type': 'content_block_start', 'index': 0, 'content_block': {**block, 'input': {}}}),
|
|
('content_block_delta', {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'input_json_delta', 'partial_json': json.dumps(value)}}),
|
|
('content_block_stop', {'type': 'content_block_stop', 'index': 0}),
|
|
('message_delta', {'type': 'message_delta', 'delta': {'stop_reason': 'tool_use', 'stop_sequence': None}, 'usage': {'output_tokens': 5}}),
|
|
('message_stop', {'type': 'message_stop'})]
|
|
out = ''.join('event: ' + k + '\ndata: ' + json.dumps(v) + '\n\n' for k, v in events).encode()
|
|
content_type = 'text/event-stream'
|
|
else:
|
|
out = json.dumps(message).encode(); content_type = 'application/json'
|
|
self.send_response(200); self.send_header('Content-Type', content_type)
|
|
self.send_header('Content-Length', str(len(out))); self.end_headers(); self.wfile.write(out)
|
|
server = HTTPServer(('127.0.0.1', 0), Mock)
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
for limit in [3, 6]:
|
|
seen.clear()
|
|
with tempfile.TemporaryDirectory(dir='/jobs') as directory:
|
|
env = claude_environment(directory, 'synthetic-unused-oauth-token')
|
|
env['ANTHROPIC_BASE_URL'] = 'http://127.0.0.1:' + str(server.server_port)
|
|
cmd = claude_command('claude-opus-4-8', 5)
|
|
cmd[cmd.index('--max-turns') + 1] = str(limit)
|
|
cmd[cmd.index('--json-schema') + 1] = json.dumps(DISCOVERY_SCHEMA)
|
|
started = time.monotonic()
|
|
p = subprocess.run(cmd, input=source, capture_output=True, text=True, env=env, cwd=directory, timeout=45)
|
|
events = [json.loads(line) for line in p.stdout.splitlines()]
|
|
final = next((e for e in reversed(events) if e.get('type') == 'result'), {})
|
|
if limit == 3:
|
|
assert p.returncode == 1 and final.get('subtype') == 'error_max_turns'
|
|
assert not final.get('structured_output')
|
|
else:
|
|
assert p.returncode == 0 and final.get('subtype') == 'success'
|
|
validate_partition(final['structured_output'], request)
|
|
assert all(item['whole_input'] and item['system_preserved'] for item in seen)
|
|
print(json.dumps({'max_turns': limit, 'exit_code': p.returncode, 'wall_seconds': round(time.monotonic()-started, 3),
|
|
'final': {k: final.get(k) for k in ('type', 'subtype', 'is_error', 'num_turns', 'stop_reason', 'usage')},
|
|
'structured_output_present': isinstance(final.get('structured_output'), dict),
|
|
'requests': seen}), flush=True)
|
|
server.shutdown()
|