docs(hermes): record 61-case diagnostic acceptance
Some checks failed
Tests / Declarative: Post Actions failed: 42, skipped: 89, passed: 4218
Some checks failed
Tests / Declarative: Post Actions failed: 42, skipped: 89, passed: 4218
This commit is contained in:
jenkins
parent
6178265f70
commit
1bc2f22bcc
333
docs/evidence/hermes_suite_61_diagnostic_20260929.json
Normal file
333
docs/evidence/hermes_suite_61_diagnostic_20260929.json
Normal file
@ -0,0 +1,333 @@
|
|||||||
|
{
|
||||||
|
"deployed_code_commit": "6178265f70c8a2abbef2b1c78bd591475eeb97a5",
|
||||||
|
"isolated_native_cli_connection_failure": {
|
||||||
|
"assistant_events": [
|
||||||
|
{
|
||||||
|
"assistant_api_error_seen": true,
|
||||||
|
"assistant_error_code": "server_error",
|
||||||
|
"exact_connection_signature": false,
|
||||||
|
"exact_timeout_signature": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"diagnostics": {
|
||||||
|
"api_error_status": null,
|
||||||
|
"assistant_api_error_seen": true,
|
||||||
|
"assistant_error_code": "server_error",
|
||||||
|
"assistant_json_text_present": false,
|
||||||
|
"assistant_structured_tool_input_present": false,
|
||||||
|
"cli_transport_error": "connection_refused",
|
||||||
|
"compaction_event_seen": false,
|
||||||
|
"configured_max_output_tokens": 64000,
|
||||||
|
"cost_usd_estimate": 0,
|
||||||
|
"duration_api_ms": 0,
|
||||||
|
"execution_revision": "suite-multipass-v8-20260929",
|
||||||
|
"exit_code": 1,
|
||||||
|
"final_event_seen": true,
|
||||||
|
"final_event_subtype": "success",
|
||||||
|
"final_event_type": "result",
|
||||||
|
"final_is_error": true,
|
||||||
|
"final_json_text_present": false,
|
||||||
|
"init_event_seen": true,
|
||||||
|
"invalid_event_count": 0,
|
||||||
|
"last_assistant_stop_reason": "stop_sequence",
|
||||||
|
"max_turns": 6,
|
||||||
|
"observed_model_limits": [],
|
||||||
|
"provider_stop_reason": "stop_sequence",
|
||||||
|
"provider_timeout_seconds": null,
|
||||||
|
"reasoning_effort": "high",
|
||||||
|
"reasoning_token_limit": null,
|
||||||
|
"structured_output_is_object": false,
|
||||||
|
"structured_output_location": null,
|
||||||
|
"structured_output_present": false,
|
||||||
|
"structured_retry_limit_reached": false,
|
||||||
|
"subprocess_timeout_seconds": null,
|
||||||
|
"termination_reason": "nonzero_exit",
|
||||||
|
"termination_signal": null,
|
||||||
|
"turn_limit_reached": false,
|
||||||
|
"turns": 1,
|
||||||
|
"usage": {
|
||||||
|
"cache_creation": {
|
||||||
|
"ephemeral_1h_input_tokens": 0,
|
||||||
|
"ephemeral_5m_input_tokens": 0
|
||||||
|
},
|
||||||
|
"cache_creation_input_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 0,
|
||||||
|
"input_tokens": 0,
|
||||||
|
"output_tokens": 0,
|
||||||
|
"output_tokens_details": {
|
||||||
|
"thinking_tokens": 0
|
||||||
|
},
|
||||||
|
"server_tool_use": {
|
||||||
|
"web_fetch_requests": 0,
|
||||||
|
"web_search_requests": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"test": "closed_loopback_connection",
|
||||||
|
"wall_seconds": 0.19
|
||||||
|
},
|
||||||
|
"live_synthetic_proposal": {
|
||||||
|
"automatic_retries": 0,
|
||||||
|
"case_count": 61,
|
||||||
|
"execution_revision": "suite-multipass-v8-20260929",
|
||||||
|
"metadata": {
|
||||||
|
"cli_diagnostics": {
|
||||||
|
"api_error_status": null,
|
||||||
|
"assistant_api_error_seen": false,
|
||||||
|
"assistant_error_code": null,
|
||||||
|
"assistant_json_text_present": false,
|
||||||
|
"assistant_structured_tool_input_present": true,
|
||||||
|
"cli_transport_error": null,
|
||||||
|
"compaction_event_seen": false,
|
||||||
|
"configured_max_output_tokens": 64000,
|
||||||
|
"cost_usd_estimate": 0.282156,
|
||||||
|
"duration_api_ms": 28321,
|
||||||
|
"execution_revision": "suite-multipass-v8-20260929",
|
||||||
|
"exit_code": 0,
|
||||||
|
"final_event_seen": true,
|
||||||
|
"final_event_subtype": "success",
|
||||||
|
"final_event_type": "result",
|
||||||
|
"final_is_error": false,
|
||||||
|
"final_json_text_present": true,
|
||||||
|
"init_event_seen": true,
|
||||||
|
"invalid_event_count": 0,
|
||||||
|
"last_assistant_stop_reason": null,
|
||||||
|
"max_turns": 6,
|
||||||
|
"observed_model_limits": [
|
||||||
|
{
|
||||||
|
"contextWindow": 1000000,
|
||||||
|
"maxOutputTokens": 128000
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"provider_stop_reason": "tool_use",
|
||||||
|
"provider_timeout_seconds": null,
|
||||||
|
"reasoning_effort": "high",
|
||||||
|
"reasoning_token_limit": null,
|
||||||
|
"structured_output_is_object": true,
|
||||||
|
"structured_output_location": "result.structured_output",
|
||||||
|
"structured_output_present": true,
|
||||||
|
"structured_retry_limit_reached": false,
|
||||||
|
"subprocess_timeout_seconds": 1799.999648113735,
|
||||||
|
"termination_reason": "exited",
|
||||||
|
"termination_signal": null,
|
||||||
|
"turn_limit_reached": false,
|
||||||
|
"turns": 3,
|
||||||
|
"usage": {
|
||||||
|
"cache_creation": {
|
||||||
|
"ephemeral_1h_input_tokens": 0,
|
||||||
|
"ephemeral_5m_input_tokens": 0
|
||||||
|
},
|
||||||
|
"cache_creation_input_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 0,
|
||||||
|
"input_tokens": 52404,
|
||||||
|
"output_tokens": 3627,
|
||||||
|
"output_tokens_details": {
|
||||||
|
"thinking_tokens": 541
|
||||||
|
},
|
||||||
|
"server_tool_use": {
|
||||||
|
"web_fetch_requests": 0,
|
||||||
|
"web_search_requests": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"compaction": false,
|
||||||
|
"compaction_signal": "CLI events and disabled compaction",
|
||||||
|
"cost_usd_estimate": 0.282156,
|
||||||
|
"duration_api_ms": 28321,
|
||||||
|
"model": "claude-opus-5-5",
|
||||||
|
"model_usage": {
|
||||||
|
"claude-opus-5-5[1m]": {
|
||||||
|
"canonicalModel": "claude-opus-5-5",
|
||||||
|
"contextWindow": 1000000,
|
||||||
|
"maxOutputTokens": 128000,
|
||||||
|
"provider": "firstParty"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"temporary_files_deleted": true,
|
||||||
|
"truncation": false,
|
||||||
|
"turns": 3,
|
||||||
|
"usage": {
|
||||||
|
"cache_creation": {
|
||||||
|
"ephemeral_1h_input_tokens": 0,
|
||||||
|
"ephemeral_5m_input_tokens": 0
|
||||||
|
},
|
||||||
|
"cache_creation_input_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 0,
|
||||||
|
"input_tokens": 52404,
|
||||||
|
"output_tokens": 3627,
|
||||||
|
"output_tokens_details": {
|
||||||
|
"thinking_tokens": 541
|
||||||
|
},
|
||||||
|
"server_tool_use": {
|
||||||
|
"web_fetch_requests": 0,
|
||||||
|
"web_search_requests": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"passes": [
|
||||||
|
{
|
||||||
|
"allocated_cost_usd": 30.0,
|
||||||
|
"allocated_seconds": 1799.999648113735,
|
||||||
|
"case_order_sha256": "b619417e9e74e83f1292f700e617bab4ec2aa051f90b5826428af402f4050344",
|
||||||
|
"cli_diagnostics": {
|
||||||
|
"api_error_status": null,
|
||||||
|
"assistant_api_error_seen": false,
|
||||||
|
"assistant_error_code": null,
|
||||||
|
"assistant_json_text_present": false,
|
||||||
|
"assistant_structured_tool_input_present": true,
|
||||||
|
"cli_transport_error": null,
|
||||||
|
"compaction_event_seen": false,
|
||||||
|
"configured_max_output_tokens": 64000,
|
||||||
|
"cost_usd_estimate": 0.282156,
|
||||||
|
"duration_api_ms": 28321,
|
||||||
|
"execution_revision": "suite-multipass-v8-20260929",
|
||||||
|
"exit_code": 0,
|
||||||
|
"final_event_seen": true,
|
||||||
|
"final_event_subtype": "success",
|
||||||
|
"final_event_type": "result",
|
||||||
|
"final_is_error": false,
|
||||||
|
"final_json_text_present": true,
|
||||||
|
"init_event_seen": true,
|
||||||
|
"invalid_event_count": 0,
|
||||||
|
"last_assistant_stop_reason": null,
|
||||||
|
"max_turns": 6,
|
||||||
|
"observed_model_limits": [
|
||||||
|
{
|
||||||
|
"contextWindow": 1000000,
|
||||||
|
"maxOutputTokens": 128000
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"provider_stop_reason": "tool_use",
|
||||||
|
"provider_timeout_seconds": null,
|
||||||
|
"reasoning_effort": "high",
|
||||||
|
"reasoning_token_limit": null,
|
||||||
|
"structured_output_is_object": true,
|
||||||
|
"structured_output_location": "result.structured_output",
|
||||||
|
"structured_output_present": true,
|
||||||
|
"structured_retry_limit_reached": false,
|
||||||
|
"subprocess_timeout_seconds": 1799.999648113735,
|
||||||
|
"termination_reason": "exited",
|
||||||
|
"termination_signal": null,
|
||||||
|
"turn_limit_reached": false,
|
||||||
|
"turns": 3,
|
||||||
|
"usage": {
|
||||||
|
"cache_creation": {
|
||||||
|
"ephemeral_1h_input_tokens": 0,
|
||||||
|
"ephemeral_5m_input_tokens": 0
|
||||||
|
},
|
||||||
|
"cache_creation_input_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 0,
|
||||||
|
"input_tokens": 52404,
|
||||||
|
"output_tokens": 3627,
|
||||||
|
"output_tokens_details": {
|
||||||
|
"thinking_tokens": 541
|
||||||
|
},
|
||||||
|
"server_tool_use": {
|
||||||
|
"web_fetch_requests": 0,
|
||||||
|
"web_search_requests": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"cli_turns": 3,
|
||||||
|
"cost_usd_estimate": 0.282156,
|
||||||
|
"duration_api_ms": 28321,
|
||||||
|
"input_bytes": 73903,
|
||||||
|
"input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer",
|
||||||
|
"input_token_bound": 82095,
|
||||||
|
"input_token_count": null,
|
||||||
|
"model": "claude-opus-5-5",
|
||||||
|
"output_reservation_tokens": 17024,
|
||||||
|
"output_reservation_verified": false,
|
||||||
|
"provider": "claude",
|
||||||
|
"reasoning": "high",
|
||||||
|
"schema_sha256": "0ea2a8d0b664468910e14af774be43b2674c2118a08f6549f066f27b7d04ebcf",
|
||||||
|
"stage": "proposal_a",
|
||||||
|
"system_sha256": "5df3f59dd1c4182d42fef8b58db69f0b18a9ff19ebaa7217363622484a29a04a",
|
||||||
|
"usage": {
|
||||||
|
"cache_creation": {
|
||||||
|
"ephemeral_1h_input_tokens": 0,
|
||||||
|
"ephemeral_5m_input_tokens": 0
|
||||||
|
},
|
||||||
|
"cache_creation_input_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 0,
|
||||||
|
"input_tokens": 52404,
|
||||||
|
"output_tokens": 3627,
|
||||||
|
"output_tokens_details": {
|
||||||
|
"thinking_tokens": 541
|
||||||
|
},
|
||||||
|
"server_tool_use": {
|
||||||
|
"web_fetch_requests": 0,
|
||||||
|
"web_search_requests": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"wall_seconds": 28.805
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"prompt_revision": "implementation-proximity-multipass-v4-20260929",
|
||||||
|
"quality": {
|
||||||
|
"coverage": true,
|
||||||
|
"false_merge_pairs": 0,
|
||||||
|
"families": 9,
|
||||||
|
"missed_merge_pairs": 0,
|
||||||
|
"pair_precision": 1.0,
|
||||||
|
"pair_recall": 1.0
|
||||||
|
},
|
||||||
|
"request_bytes": 64502,
|
||||||
|
"selection": {
|
||||||
|
"backend": "claude-code-2.1.285",
|
||||||
|
"case_count": 61,
|
||||||
|
"cli_model": "claude-opus-5-5[1m]",
|
||||||
|
"configuration_revision": "suite-v6-20260929",
|
||||||
|
"context": 1000000,
|
||||||
|
"enabled": true,
|
||||||
|
"execution_revision": "suite-multipass-v8-20260929",
|
||||||
|
"input_bytes": 73903,
|
||||||
|
"input_count_method": "Complete UTF-8 input/system/schema byte bound plus harness overhead; not a tokenizer",
|
||||||
|
"input_token_bound": 82095,
|
||||||
|
"input_token_count": null,
|
||||||
|
"later_pass_capacity_verified": false,
|
||||||
|
"later_pass_checks": "before_each_invocation",
|
||||||
|
"max_final_group_cases": 5,
|
||||||
|
"max_final_name_characters": 64,
|
||||||
|
"max_turns": 6,
|
||||||
|
"maximum_model_passes": 5,
|
||||||
|
"minimum_model_passes": 3,
|
||||||
|
"model": "claude-opus-5-5",
|
||||||
|
"output": 64000,
|
||||||
|
"output_reservation_tokens": 17024,
|
||||||
|
"output_reservation_verified": false,
|
||||||
|
"overhead": 8192,
|
||||||
|
"policy_revision": "implementation-five-v1-20260929",
|
||||||
|
"prompt_revision": "implementation-proximity-multipass-v4-20260929",
|
||||||
|
"prompt_sha256": "253314b009ea82fc44695260ece8edd501ab60dc40ab3cc9e1ac9a0acb2017c0",
|
||||||
|
"provider": "claude",
|
||||||
|
"reasoning": "high",
|
||||||
|
"reasoning_policy": {
|
||||||
|
"case_count_at_least": 100,
|
||||||
|
"default": "high",
|
||||||
|
"large": "xhigh",
|
||||||
|
"match": "any",
|
||||||
|
"revision": "suite-size-effort-v1-20260929",
|
||||||
|
"scope": "whole_job",
|
||||||
|
"source_bytes_at_least": 131072,
|
||||||
|
"source_bytes_scope": "Canonical UTF-8 JSON of campaign, suite, and all cases"
|
||||||
|
},
|
||||||
|
"reasoning_selection": {
|
||||||
|
"case_count": 61,
|
||||||
|
"policy_revision": "suite-size-effort-v1-20260929",
|
||||||
|
"scope": "whole_job",
|
||||||
|
"source_bytes": 64350,
|
||||||
|
"triggers": []
|
||||||
|
},
|
||||||
|
"reported_output": 128000,
|
||||||
|
"source_sha256": "316cc6ec17dafca611a802eb070b5929b8dccfb91bca6208c917c25e77661055"
|
||||||
|
},
|
||||||
|
"status": "completed",
|
||||||
|
"test": "synthetic_proposal_a_only",
|
||||||
|
"wall_seconds": 28.806
|
||||||
|
},
|
||||||
|
"scope": "Synthetic server-side proposal_a only; not a complete multi-pass job or laptop connectivity test",
|
||||||
|
"unit_regression_tests": {
|
||||||
|
"passed": 149
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -57,9 +57,24 @@ The real job selected `high`; v8 reports the actual invocation configuration.
|
|||||||
specific historical upstream failure. They used fake credentials and no
|
specific historical upstream failure. They used fake credentials and no
|
||||||
hosted inference.
|
hosted inference.
|
||||||
|
|
||||||
The separately recorded live synthetic check exercises one complete 61-case
|
The deployed live synthetic check completed one 61-case `proposal_a` on
|
||||||
proposal, not the full multi-pass workflow. It must not be represented as a
|
`claude-opus-5-5` at high effort in 28.806 seconds, using three of six turns.
|
||||||
rerun or recovery of the failed real job.
|
The serialized request was 64502 bytes (64350 bytes of source content;
|
||||||
|
73903 bytes including pass instructions/schema). The CLI reported 52404 input
|
||||||
|
and 3627 output tokens across the invocation, including 541 thinking tokens,
|
||||||
|
and a $0.282156 cost estimate (not a subscription billing statement).
|
||||||
|
|
||||||
|
Strict validation accepted all 61 aliases exactly once. All nine natural
|
||||||
|
families matched the synthetic fixture, with zero false/missed merge pairs.
|
||||||
|
The terminal event had `is_error: false`, exit 0, and an object at
|
||||||
|
`result.structured_output`. No compaction event or truncation was detected.
|
||||||
|
There were no operator retries. This exercises one complete proposal, not the
|
||||||
|
full multi-pass workflow, and does not establish the original upstream cause.
|
||||||
|
|
||||||
|
Flux applied code commit `6178265f70c8a2abbef2b1c78bd591475eeb97a5` and the
|
||||||
|
planner rolled out successfully. Authenticated in-pod preflight reports v8;
|
||||||
|
this is not a new laptop/LAN connectivity check. Measured safe evidence is in
|
||||||
|
`docs/evidence/hermes_suite_61_diagnostic_20260929.json`.
|
||||||
|
|
||||||
## Client handling and rollback
|
## Client handling and rollback
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user