diff --git a/docs/evidence/hermes_suite_61_diagnostic_20260929.json b/docs/evidence/hermes_suite_61_diagnostic_20260929.json new file mode 100644 index 00000000..0436d2ef --- /dev/null +++ b/docs/evidence/hermes_suite_61_diagnostic_20260929.json @@ -0,0 +1,333 @@ +{ + "deployed_code_commit": "6178265f70c8a2abbef2b1c78bd591475eeb97a5", + "isolated_native_cli_connection_failure": { + "assistant_events": [ + { + "assistant_api_error_seen": true, + "assistant_error_code": "server_error", + "exact_connection_signature": false, + "exact_timeout_signature": false + } + ], + "diagnostics": { + "api_error_status": null, + "assistant_api_error_seen": true, + "assistant_error_code": "server_error", + "assistant_json_text_present": false, + "assistant_structured_tool_input_present": false, + "cli_transport_error": "connection_refused", + "compaction_event_seen": false, + "configured_max_output_tokens": 64000, + "cost_usd_estimate": 0, + "duration_api_ms": 0, + "execution_revision": "suite-multipass-v8-20260929", + "exit_code": 1, + "final_event_seen": true, + "final_event_subtype": "success", + "final_event_type": "result", + "final_is_error": true, + "final_json_text_present": false, + "init_event_seen": true, + "invalid_event_count": 0, + "last_assistant_stop_reason": "stop_sequence", + "max_turns": 6, + "observed_model_limits": [], + "provider_stop_reason": "stop_sequence", + "provider_timeout_seconds": null, + "reasoning_effort": "high", + "reasoning_token_limit": null, + "structured_output_is_object": false, + "structured_output_location": null, + "structured_output_present": false, + "structured_retry_limit_reached": false, + "subprocess_timeout_seconds": null, + "termination_reason": "nonzero_exit", + "termination_signal": null, + "turn_limit_reached": false, + "turns": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 0, + "output_tokens": 0, + "output_tokens_details": { + "thinking_tokens": 0 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + } + } + }, + "test": "closed_loopback_connection", + "wall_seconds": 0.19 + }, + "live_synthetic_proposal": { + "automatic_retries": 0, + "case_count": 61, + "execution_revision": "suite-multipass-v8-20260929", + "metadata": { + "cli_diagnostics": { + "api_error_status": null, + "assistant_api_error_seen": false, + "assistant_error_code": null, + "assistant_json_text_present": false, + "assistant_structured_tool_input_present": true, + "cli_transport_error": null, + "compaction_event_seen": false, + "configured_max_output_tokens": 64000, + "cost_usd_estimate": 0.282156, + "duration_api_ms": 28321, + "execution_revision": "suite-multipass-v8-20260929", + "exit_code": 0, + "final_event_seen": true, + "final_event_subtype": "success", + "final_event_type": "result", + "final_is_error": false, + "final_json_text_present": true, + "init_event_seen": true, + "invalid_event_count": 0, + "last_assistant_stop_reason": null, + "max_turns": 6, + "observed_model_limits": [ + { + "contextWindow": 1000000, + "maxOutputTokens": 128000 + } + ], + "provider_stop_reason": "tool_use", + "provider_timeout_seconds": null, + "reasoning_effort": "high", + "reasoning_token_limit": null, + "structured_output_is_object": true, + "structured_output_location": "result.structured_output", + "structured_output_present": true, + "structured_retry_limit_reached": false, + "subprocess_timeout_seconds": 1799.999648113735, + "termination_reason": "exited", + "termination_signal": null, + "turn_limit_reached": false, + "turns": 3, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 52404, + "output_tokens": 3627, + "output_tokens_details": { + "thinking_tokens": 541 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + } + } + }, + "compaction": false, + "compaction_signal": "CLI events and disabled compaction", + "cost_usd_estimate": 0.282156, + "duration_api_ms": 28321, + "model": "claude-opus-5-5", + "model_usage": { + "claude-opus-5-5[1m]": { + "canonicalModel": "claude-opus-5-5", + "contextWindow": 1000000, + "maxOutputTokens": 128000, + "provider": "firstParty" + } + }, + "temporary_files_deleted": true, + "truncation": false, + "turns": 3, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 52404, + "output_tokens": 3627, + "output_tokens_details": { + "thinking_tokens": 541 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + } + } + }, + "passes": [ + { + "allocated_cost_usd": 30.0, + "allocated_seconds": 1799.999648113735, + "case_order_sha256": "b619417e9e74e83f1292f700e617bab4ec2aa051f90b5826428af402f4050344", + "cli_diagnostics": { + "api_error_status": null, + "assistant_api_error_seen": false, + "assistant_error_code": null, + "assistant_json_text_present": false, + "assistant_structured_tool_input_present": true, + "cli_transport_error": null, + "compaction_event_seen": false, + "configured_max_output_tokens": 64000, + "cost_usd_estimate": 0.282156, + "duration_api_ms": 28321, + "execution_revision": "suite-multipass-v8-20260929", + "exit_code": 0, + "final_event_seen": true, + "final_event_subtype": "success", + "final_event_type": "result", + "final_is_error": false, + "final_json_text_present": true, + "init_event_seen": true, + "invalid_event_count": 0, + "last_assistant_stop_reason": null, + "max_turns": 6, + "observed_model_limits": [ + { + "contextWindow": 1000000, + "maxOutputTokens": 128000 + } + ], + "provider_stop_reason": "tool_use", + "provider_timeout_seconds": null, + "reasoning_effort": "high", + "reasoning_token_limit": null, + "structured_output_is_object": true, + "structured_output_location": "result.structured_output", + "structured_output_present": true, + "structured_retry_limit_reached": false, + "subprocess_timeout_seconds": 1799.999648113735, + "termination_reason": "exited", + "termination_signal": null, + "turn_limit_reached": false, + "turns": 3, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 52404, + "output_tokens": 3627, + "output_tokens_details": { + "thinking_tokens": 541 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + } + } + }, + "cli_turns": 3, + "cost_usd_estimate": 0.282156, + "duration_api_ms": 28321, + "input_bytes": 73903, + "input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer", + "input_token_bound": 82095, + "input_token_count": null, + "model": "claude-opus-5-5", + "output_reservation_tokens": 17024, + "output_reservation_verified": false, + "provider": "claude", + "reasoning": "high", + "schema_sha256": "0ea2a8d0b664468910e14af774be43b2674c2118a08f6549f066f27b7d04ebcf", + "stage": "proposal_a", + "system_sha256": "5df3f59dd1c4182d42fef8b58db69f0b18a9ff19ebaa7217363622484a29a04a", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 52404, + "output_tokens": 3627, + "output_tokens_details": { + "thinking_tokens": 541 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + } + }, + "wall_seconds": 28.805 + } + ], + "prompt_revision": "implementation-proximity-multipass-v4-20260929", + "quality": { + "coverage": true, + "false_merge_pairs": 0, + "families": 9, + "missed_merge_pairs": 0, + "pair_precision": 1.0, + "pair_recall": 1.0 + }, + "request_bytes": 64502, + "selection": { + "backend": "claude-code-2.1.285", + "case_count": 61, + "cli_model": "claude-opus-5-5[1m]", + "configuration_revision": "suite-v6-20260929", + "context": 1000000, + "enabled": true, + "execution_revision": "suite-multipass-v8-20260929", + "input_bytes": 73903, + "input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer", + "input_token_bound": 82095, + "input_token_count": null, + "later_pass_capacity_verified": false, + "later_pass_checks": "before_each_invocation", + "max_final_group_cases": 5, + "max_final_name_characters": 64, + "max_turns": 6, + "maximum_model_passes": 5, + "minimum_model_passes": 3, + "model": "claude-opus-5-5", + "output": 64000, + "output_reservation_tokens": 17024, + "output_reservation_verified": false, + "overhead": 8192, + "policy_revision": "implementation-five-v1-20260929", + "prompt_revision": "implementation-proximity-multipass-v4-20260929", + "prompt_sha256": "253314b009ea82fc44695260ece8edd501ab60dc40ab3cc9e1ac9a0acb2017c0", + "provider": "claude", + "reasoning": "high", + "reasoning_policy": { + "case_count_at_least": 100, + "default": "high", + "large": "xhigh", + "match": "any", + "revision": "suite-size-effort-v1-20260929", + "scope": "whole_job", + "source_bytes_at_least": 131072, + "source_bytes_scope": "Canonical UTF-8 JSON of campaign, suite, and all cases" + }, + "reasoning_selection": { + "case_count": 61, + "policy_revision": "suite-size-effort-v1-20260929", + "scope": "whole_job", + "source_bytes": 64350, + "triggers": [] + }, + "reported_output": 128000, + "source_sha256": "316cc6ec17dafca611a802eb070b5929b8dccfb91bca6208c917c25e77661055" + }, + "status": "completed", + "test": "synthetic_proposal_a_only", + "wall_seconds": 28.806 + }, + "scope": "Synthetic server-side proposal_a only; not a complete multi-pass job or laptop connectivity test", + "unit_regression_tests": { + "passed": 149 + } +} diff --git a/docs/hermes_suite_61_failure_20260929.md b/docs/hermes_suite_61_failure_20260929.md index bb11a42c..f8d0b5b4 100644 --- a/docs/hermes_suite_61_failure_20260929.md +++ b/docs/hermes_suite_61_failure_20260929.md @@ -57,9 +57,24 @@ The real job selected `high`; v8 reports the actual invocation configuration. specific historical upstream failure. They used fake credentials and no hosted inference. -The separately recorded live synthetic check exercises one complete 61-case -proposal, not the full multi-pass workflow. It must not be represented as a -rerun or recovery of the failed real job. +The deployed live synthetic check completed one 61-case `proposal_a` on +`claude-opus-5-5` at high effort in 28.806 seconds, using three of six turns. +The serialized request was 64502 bytes (64350 bytes of source content; +73903 bytes including pass instructions/schema). The CLI reported 52404 input +and 3627 output tokens across the invocation, including 541 thinking tokens, +and a $0.282156 cost estimate (not a subscription billing statement). + +Strict validation accepted all 61 aliases exactly once. All nine natural +families matched the synthetic fixture, with zero false/missed merge pairs. +The terminal event had `is_error: false`, exit 0, and an object at +`result.structured_output`. No compaction event or truncation was detected. +There were no operator retries. This exercises one complete proposal, not the +full multi-pass workflow, and does not establish the original upstream cause. + +Flux applied code commit `6178265f70c8a2abbef2b1c78bd591475eeb97a5` and the +planner rolled out successfully. Authenticated in-pod preflight reports v8; +this is not a new laptop/LAN connectivity check. Measured safe evidence is in +`docs/evidence/hermes_suite_61_diagnostic_20260929.json`. ## Client handling and rollback