atlas-iac/scripts/ops/hermes_suite_cli_probe.py

#!/usr/bin/env python3
"""Synthetic native CLI turn-limit probe; all model HTTP goes to loopback.

Run inside the planner with: kubectl exec -i ... -- python < this-file
No hosted inference or live account credential is used.
"""
import json, threading, tempfile, pathlib, subprocess, sys, time
from http.server import HTTPServer, BaseHTTPRequestHandler
sys.path.insert(0, '/opt/planner')
from suite_backends import claude_environment, claude_command
from suite_contract import SYSTEM, prompt, validate_partition
from suite_policy import DISCOVERY_SCHEMA
from suite_synthetic import fixture
seen = []
request = fixture(14)[0]
source = prompt(request)
def strings(value):
    if isinstance(value, str): yield value
    elif isinstance(value, list):
        for item in value: yield from strings(item)
    elif isinstance(value, dict):
        for item in value.values(): yield from strings(item)
class Mock(BaseHTTPRequestHandler):
    def log_message(self, *args): pass
    def do_POST(self):
        body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
        seen.append({'stream': body.get('stream'), 'max_tokens': body.get('max_tokens'),
                     'http_timeout_header': self.headers.get('x-stainless-timeout'),
                     'whole_input': source in list(strings(body.get('messages', []))),
                     'system_preserved': any(SYSTEM in s for s in strings(body.get('system', [])))})
        value = {} if len(seen) <= 3 else {'groups': [{'name': 'Synthetic', 'description': 'Mock only', 'members': [c['alias'] for c in request['cases']]}]}
        block = {'type': 'tool_use', 'id': 'mock-' + str(len(seen)), 'name': 'StructuredOutput', 'input': value}
        message = {'id': 'mock', 'type': 'message', 'role': 'assistant', 'model': 'claude-opus-4-8',
                   'content': [block], 'stop_reason': 'tool_use', 'stop_sequence': None,
                   'usage': {'input_tokens': 10, 'output_tokens': 5}}
        if body.get('stream'):
            initial = {**message, 'content': [], 'stop_reason': None, 'usage': {'input_tokens': 10, 'output_tokens': 0}}
            events = [('message_start', {'type': 'message_start', 'message': initial}),
                      ('content_block_start', {'type': 'content_block_start', 'index': 0, 'content_block': {**block, 'input': {}}}),
                      ('content_block_delta', {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'input_json_delta', 'partial_json': json.dumps(value)}}),
                      ('content_block_stop', {'type': 'content_block_stop', 'index': 0}),
                      ('message_delta', {'type': 'message_delta', 'delta': {'stop_reason': 'tool_use', 'stop_sequence': None}, 'usage': {'output_tokens': 5}}),
                      ('message_stop', {'type': 'message_stop'})]
            out = ''.join('event: ' + k + '\ndata: ' + json.dumps(v) + '\n\n' for k, v in events).encode()
            content_type = 'text/event-stream'
        else:
            out = json.dumps(message).encode(); content_type = 'application/json'
        self.send_response(200); self.send_header('Content-Type', content_type)
        self.send_header('Content-Length', str(len(out))); self.end_headers(); self.wfile.write(out)
server = HTTPServer(('127.0.0.1', 0), Mock)
threading.Thread(target=server.serve_forever, daemon=True).start()
for limit in [3, 6]:
    seen.clear()
    with tempfile.TemporaryDirectory(dir='/jobs') as directory:
        env = claude_environment(directory, 'synthetic-unused-oauth-token')
        env['ANTHROPIC_BASE_URL'] = 'http://127.0.0.1:' + str(server.server_port)
        cmd = claude_command('claude-opus-4-8', 5)
        cmd[cmd.index('--max-turns') + 1] = str(limit)
        cmd[cmd.index('--json-schema') + 1] = json.dumps(DISCOVERY_SCHEMA)
        started = time.monotonic()
        p = subprocess.run(cmd, input=source, capture_output=True, text=True, env=env, cwd=directory, timeout=45)
        events = [json.loads(line) for line in p.stdout.splitlines()]
        final = next((e for e in reversed(events) if e.get('type') == 'result'), {})
        if limit == 3:
            assert p.returncode == 1 and final.get('subtype') == 'error_max_turns'
            assert not final.get('structured_output')
        else:
            assert p.returncode == 0 and final.get('subtype') == 'success'
            validate_partition(final['structured_output'], request)
        assert all(item['whole_input'] and item['system_preserved'] for item in seen)
        print(json.dumps({'max_turns': limit, 'exit_code': p.returncode, 'wall_seconds': round(time.monotonic()-started, 3),
                          'final': {k: final.get(k) for k in ('type', 'subtype', 'is_error', 'num_turns', 'stop_reason', 'usage')},
                          'structured_output_present': isinstance(final.get('structured_output'), dict),
                          'requests': seen}), flush=True)
server.shutdown()