titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

docs(hermes): record 61-case diagnostic acceptance
Some checks failed
Tests / Declarative: Post Actions failed: 42, skipped: 89, passed: 4218
Details

Browse Source
This commit is contained in:
jenkins 2026-09-29 16:48:41 -05:00
parent 6178265f70
commit 1bc2f22bcc
2 changed files with 351 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

333
docs/evidence/hermes_suite_61_diagnostic_20260929.json Normal file
View File

@ -0,0 +1,333 @@
{
"deployed_code_commit": "6178265f70c8a2abbef2b1c78bd591475eeb97a5",
"isolated_native_cli_connection_failure": {
"assistant_events": [
{
"assistant_api_error_seen": true,
"assistant_error_code": "server_error",
"exact_connection_signature": false,
"exact_timeout_signature": false
}
],
"diagnostics": {
"api_error_status": null,
"assistant_api_error_seen": true,
"assistant_error_code": "server_error",
"assistant_json_text_present": false,
"assistant_structured_tool_input_present": false,
"cli_transport_error": "connection_refused",
"compaction_event_seen": false,
"configured_max_output_tokens": 64000,
"cost_usd_estimate": 0,
"duration_api_ms": 0,
"execution_revision": "suite-multipass-v8-20260929",
"exit_code": 1,
"final_event_seen": true,
"final_event_subtype": "success",
"final_event_type": "result",
"final_is_error": true,
"final_json_text_present": false,
"init_event_seen": true,
"invalid_event_count": 0,
"last_assistant_stop_reason": "stop_sequence",
"max_turns": 6,
"observed_model_limits": [],
"provider_stop_reason": "stop_sequence",
"provider_timeout_seconds": null,
"reasoning_effort": "high",
"reasoning_token_limit": null,
"structured_output_is_object": false,
"structured_output_location": null,
"structured_output_present": false,
"structured_retry_limit_reached": false,
"subprocess_timeout_seconds": null,
"termination_reason": "nonzero_exit",
"termination_signal": null,
"turn_limit_reached": false,
"turns": 1,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"input_tokens": 0,
"output_tokens": 0,
"output_tokens_details": {
"thinking_tokens": 0
},
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
}
}
},
"test": "closed_loopback_connection",
"wall_seconds": 0.19
},
"live_synthetic_proposal": {
"automatic_retries": 0,
"case_count": 61,
"execution_revision": "suite-multipass-v8-20260929",
"metadata": {
"cli_diagnostics": {
"api_error_status": null,
"assistant_api_error_seen": false,
"assistant_error_code": null,
"assistant_json_text_present": false,
"assistant_structured_tool_input_present": true,
"cli_transport_error": null,
"compaction_event_seen": false,
"configured_max_output_tokens": 64000,
"cost_usd_estimate": 0.282156,
"duration_api_ms": 28321,
"execution_revision": "suite-multipass-v8-20260929",
"exit_code": 0,
"final_event_seen": true,
"final_event_subtype": "success",
"final_event_type": "result",
"final_is_error": false,
"final_json_text_present": true,
"init_event_seen": true,
"invalid_event_count": 0,
"last_assistant_stop_reason": null,
"max_turns": 6,
"observed_model_limits": [
{
"contextWindow": 1000000,
"maxOutputTokens": 128000
}
],
"provider_stop_reason": "tool_use",
"provider_timeout_seconds": null,
"reasoning_effort": "high",
"reasoning_token_limit": null,
"structured_output_is_object": true,
"structured_output_location": "result.structured_output",
"structured_output_present": true,
"structured_retry_limit_reached": false,
"subprocess_timeout_seconds": 1799.999648113735,
"termination_reason": "exited",
"termination_signal": null,
"turn_limit_reached": false,
"turns": 3,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"input_tokens": 52404,
"output_tokens": 3627,
"output_tokens_details": {
"thinking_tokens": 541
},
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
}
}
},
"compaction": false,
"compaction_signal": "CLI events and disabled compaction",
"cost_usd_estimate": 0.282156,
"duration_api_ms": 28321,
"model": "claude-opus-5-5",
"model_usage": {
"claude-opus-5-5[1m]": {
"canonicalModel": "claude-opus-5-5",
"contextWindow": 1000000,
"maxOutputTokens": 128000,
"provider": "firstParty"
}
},
"temporary_files_deleted": true,
"truncation": false,
"turns": 3,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"input_tokens": 52404,
"output_tokens": 3627,
"output_tokens_details": {
"thinking_tokens": 541
},
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
}
}
},
"passes": [
{
"allocated_cost_usd": 30.0,
"allocated_seconds": 1799.999648113735,
"case_order_sha256": "b619417e9e74e83f1292f700e617bab4ec2aa051f90b5826428af402f4050344",
"cli_diagnostics": {
"api_error_status": null,
"assistant_api_error_seen": false,
"assistant_error_code": null,
"assistant_json_text_present": false,
"assistant_structured_tool_input_present": true,
"cli_transport_error": null,
"compaction_event_seen": false,
"configured_max_output_tokens": 64000,
"cost_usd_estimate": 0.282156,
"duration_api_ms": 28321,
"execution_revision": "suite-multipass-v8-20260929",
"exit_code": 0,
"final_event_seen": true,
"final_event_subtype": "success",
"final_event_type": "result",
"final_is_error": false,
"final_json_text_present": true,
"init_event_seen": true,
"invalid_event_count": 0,
"last_assistant_stop_reason": null,
"max_turns": 6,
"observed_model_limits": [
{
"contextWindow": 1000000,
"maxOutputTokens": 128000
}
],
"provider_stop_reason": "tool_use",
"provider_timeout_seconds": null,
"reasoning_effort": "high",
"reasoning_token_limit": null,
"structured_output_is_object": true,
"structured_output_location": "result.structured_output",
"structured_output_present": true,
"structured_retry_limit_reached": false,
"subprocess_timeout_seconds": 1799.999648113735,
"termination_reason": "exited",
"termination_signal": null,
"turn_limit_reached": false,
"turns": 3,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"input_tokens": 52404,
"output_tokens": 3627,
"output_tokens_details": {
"thinking_tokens": 541
},
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
}
}
},
"cli_turns": 3,
"cost_usd_estimate": 0.282156,
"duration_api_ms": 28321,
"input_bytes": 73903,
"input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer",
"input_token_bound": 82095,
"input_token_count": null,
"model": "claude-opus-5-5",
"output_reservation_tokens": 17024,
"output_reservation_verified": false,
"provider": "claude",
"reasoning": "high",
"schema_sha256": "0ea2a8d0b664468910e14af774be43b2674c2118a08f6549f066f27b7d04ebcf",
"stage": "proposal_a",
"system_sha256": "5df3f59dd1c4182d42fef8b58db69f0b18a9ff19ebaa7217363622484a29a04a",
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"input_tokens": 52404,
"output_tokens": 3627,
"output_tokens_details": {
"thinking_tokens": 541
},
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
}
},
"wall_seconds": 28.805
}
],
"prompt_revision": "implementation-proximity-multipass-v4-20260929",
"quality": {
"coverage": true,
"false_merge_pairs": 0,
"families": 9,
"missed_merge_pairs": 0,
"pair_precision": 1.0,
"pair_recall": 1.0
},
"request_bytes": 64502,
"selection": {
"backend": "claude-code-2.1.285",
"case_count": 61,
"cli_model": "claude-opus-5-5[1m]",
"configuration_revision": "suite-v6-20260929",
"context": 1000000,
"enabled": true,
"execution_revision": "suite-multipass-v8-20260929",
"input_bytes": 73903,
"input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer",
"input_token_bound": 82095,
"input_token_count": null,
"later_pass_capacity_verified": false,
"later_pass_checks": "before_each_invocation",
"max_final_group_cases": 5,
"max_final_name_characters": 64,
"max_turns": 6,
"maximum_model_passes": 5,
"minimum_model_passes": 3,
"model": "claude-opus-5-5",
"output": 64000,
"output_reservation_tokens": 17024,
"output_reservation_verified": false,
"overhead": 8192,
"policy_revision": "implementation-five-v1-20260929",
"prompt_revision": "implementation-proximity-multipass-v4-20260929",
"prompt_sha256": "253314b009ea82fc44695260ece8edd501ab60dc40ab3cc9e1ac9a0acb2017c0",
"provider": "claude",
"reasoning": "high",
"reasoning_policy": {
"case_count_at_least": 100,
"default": "high",
"large": "xhigh",
"match": "any",
"revision": "suite-size-effort-v1-20260929",
"scope": "whole_job",
"source_bytes_at_least": 131072,
"source_bytes_scope": "Canonical UTF-8 JSON of campaign, suite, and all cases"
},
"reasoning_selection": {
"case_count": 61,
"policy_revision": "suite-size-effort-v1-20260929",
"scope": "whole_job",
"source_bytes": 64350,
"triggers": []
},
"reported_output": 128000,
"source_sha256": "316cc6ec17dafca611a802eb070b5929b8dccfb91bca6208c917c25e77661055"
},
"status": "completed",
"test": "synthetic_proposal_a_only",
"wall_seconds": 28.806
},
"scope": "Synthetic server-side proposal_a only; not a complete multi-pass job or laptop connectivity test",
"unit_regression_tests": {
"passed": 149
}
}

21
docs/hermes_suite_61_failure_20260929.md
View File

@ -57,9 +57,24 @@ The real job selected `high`; v8 reports the actual invocation configuration.
specific historical upstream failure. They used fake credentials and no
hosted inference.
The separately recorded live synthetic check exercises one complete 61-case
proposal, not the full multi-pass workflow. It must not be represented as a
rerun or recovery of the failed real job.
The deployed live synthetic check completed one 61-case `proposal_a` on
`claude-opus-5-5` at high effort in 28.806 seconds, using three of six turns.
The serialized request was 64502 bytes (64350 bytes of source content;
73903 bytes including pass instructions/schema). The CLI reported 52404 input
and 3627 output tokens across the invocation, including 541 thinking tokens,
and a $0.282156 cost estimate (not a subscription billing statement).
Strict validation accepted all 61 aliases exactly once. All nine natural
families matched the synthetic fixture, with zero false/missed merge pairs.
The terminal event had `is_error: false`, exit 0, and an object at
`result.structured_output`. No compaction event or truncation was detected.
There were no operator retries. This exercises one complete proposal, not the
full multi-pass workflow, and does not establish the original upstream cause.
Flux applied code commit `6178265f70c8a2abbef2b1c78bd591475eeb97a5` and the
planner rolled out successfully. Authenticated in-pod preflight reports v8;
this is not a new laptop/LAN connectivity check. Measured safe evidence is in
`docs/evidence/hermes_suite_61_diagnostic_20260929.json`.
## Client handling and rollback