From 5f58df0f03ec579569f0dac9833dd5c6d8c52334 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 29 Sep 2026 11:04:04 -0500 Subject: [PATCH] docs: record implementation-proximity prompt acceptance --- .../hermes_suite_prompt_v2_20260929.json | 182 ++++++++++++++++++ docs/hermes_suite_planning.md | 11 ++ 2 files changed, 193 insertions(+) create mode 100644 docs/evidence/hermes_suite_prompt_v2_20260929.json diff --git a/docs/evidence/hermes_suite_prompt_v2_20260929.json b/docs/evidence/hermes_suite_prompt_v2_20260929.json new file mode 100644 index 00000000..085d126e --- /dev/null +++ b/docs/evidence/hermes_suite_prompt_v2_20260929.json @@ -0,0 +1,182 @@ +{ + "configuration_revision": "suite-v6-20260929", + "prompt_revision": "implementation-proximity-v2-20260929", + "prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552", + "deployed_commit": "a7319ea03b727a159606871cf73b8e6f29b0f757", + "synthetic_only": true, + "fixture": "testing/fixtures/suite_implementation_proximity.json", + "expected_answers_sent": false, + "quality": { + "coverage": true, + "families": 6, + "pair_precision": 1.0, + "pair_recall": 1.0, + "false_merge_pairs": 0, + "missed_merge_pairs": 0 + }, + "description_review": "No observed cross-family objectives. Shared JSON assertions and waveform-derived duration stay with their reusable machinery; absent interface/measurement details remain explicit uncertainty.", + "cli_prompt_propagation": [ + { + "cases": 14, + "requested": "claude-opus-4-8[1m]", + "captured": [ + { + "path": "/v1/messages?beta=true", + "model": "claude-opus-4-8", + "max_tokens": 64000, + "bytes": 15617, + "full_grouping_system_prompt": true, + "complete_source": true + }, + { + "path": "/v1/messages?beta=true", + "model": "claude-opus-4-8", + "max_tokens": 64000, + "bytes": 15603, + "full_grouping_system_prompt": true, + "complete_source": true + } + ], + "returncode": 0 + } + ], + "acceptance": { + "client_wall_seconds": 21.059, + "job": { + "attempted_destinations": [ + "switchyard:atlas/planning/claude", + "claude:claude-opus-4-8" + ], + "compaction": false, + "compaction_signal": "CLI events and disabled compaction", + "configuration_revision": "suite-v6-20260929", + "cost_usd_estimate": 0.10087499999999999, + "created_at": 1790697742.2365413, + "duration_api_ms": 19807, + "job_id": "661aefd2d1f14579a264a34d64bf704b", + "model": "claude-opus-4-8", + "model_usage": { + "claude-opus-4-8[1m]": { + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "canonicalModel": "claude-opus-4-8", + "contextWindow": 1000000, + "costUSD": 0.10087499999999999, + "inputTokens": 11010, + "maxOutputTokens": 64000, + "outputTokens": 1833, + "provider": "firstParty", + "webSearchRequests": 0 + } + }, + "prompt_revision": "implementation-proximity-v2-20260929", + "prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552", + "result": { + "groups": [ + { + "description": "Issue mode via command/readback JSON client with reset-state fixture, collect JSON snapshot, assert acceptance flag and reported state. Variants differ in true/false acceptance, added error_code field, values\u2014input/assertion variations.", + "members": [ + "CASE-0001", + "CASE-0005", + "CASE-0009", + "CASE-0013" + ], + "name": "JSON command/readback client mode assertions" + }, + { + "description": "Apply mode transition via bench controller, capture output waveform with oscilloscope adapter/probe, assert amplitude and transition ordering. One variant also derives transition duration from captured samples; same capture mechanism.", + "members": [ + "CASE-0002", + "CASE-0006", + "CASE-0010", + "CASE-0014" + ], + "name": "Oscilloscope waveform capture mode-transition tests" + }, + { + "description": "Parse offline static-analysis report with shared report parser and rule-policy fixture (no target execution), count findings per rule/severity, compare to qualitative acceptance bound. Variants differ only in inputs/bounds.", + "members": [ + "CASE-0003", + "CASE-0007", + "CASE-0011" + ], + "name": "Static-analysis report parsing finding-count checks" + }, + { + "description": "Cycle chamber temperature and measure enclosure expansion with a calibrated dimensional gauge, asserting expansion below the generalized bound. Distinct thermal/gauge machinery independent of waveform bench; single case.", + "members": [ + "CASE-0004" + ], + "name": "Thermal-chamber enclosure expansion measurement" + }, + { + "description": "Record acoustic output in anechoic enclosure via microphone capture and spectral-analysis fixture; assert each spectral peak below a frequency-dependent bound. Unique acoustic instrumentation not shared with other families; single case.", + "members": [ + "CASE-0008" + ], + "name": "Acoustic spectral-peak measurement" + }, + { + "description": "Verify a latent-state relationship where required interface, stimulus, observation mechanism, and fixture are all omitted. Insufficient evidence to determine machinery or merge; retained as standalone uncertainty.", + "members": [ + "CASE-0012" + ], + "name": "Underspecified latent-state verification" + } + ] + }, + "result_retention_seconds": 3600, + "routing": { + "allow_external": true, + "allowed_external_providers": [ + "claude" + ] + }, + "selection": { + "backend": "claude-code-2.1.226", + "case_count": 14, + "cli_model": "claude-opus-4-8[1m]", + "configuration_revision": "suite-v6-20260929", + "context": 1000000, + "enabled": true, + "input_bytes": 13568, + "input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer", + "input_token_bound": 21760, + "input_token_count": null, + "model": "claude-opus-4-8", + "output": 64000, + "output_reservation_tokens": 10910, + "output_reservation_verified": false, + "overhead": 8192, + "prompt_revision": "implementation-proximity-v2-20260929", + "prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552", + "provider": "claude", + "reasoning": "medium", + "source_sha256": "37c6ca89c6c8fb170b104b5aec4e5aae65463e5d74fe284ab7ef1ecb55a0aac3" + }, + "status": "completed", + "temporary_files_deleted": true, + "truncation": false, + "turns": 3, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "not_available", + "input_tokens": 11010, + "iterations": [], + "output_tokens": 1833, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + }, + "wall_seconds": 20.544 + } + } +} diff --git a/docs/hermes_suite_planning.md b/docs/hermes_suite_planning.md index 0b7550f9..20d75a72 100644 --- a/docs/hermes_suite_planning.md +++ b/docs/hermes_suite_planning.md @@ -36,6 +36,17 @@ The synthetic check fixture is sent, never its independent expected-family mapping. No real roster job is launched by this deployment or its acceptance checks. +Verified on the deployed endpoint: the 14-case adversarial synthetic suite completed +in 21.059 seconds with six implementation families, exact alias coverage, and zero +incorrect or missed merge pairs against its independent expected map. Identical +broad descriptions did not merge different machinery. Nominal/negative-result and +extra-assertion variants shared their existing observation code; an underspecified +case retained its uncertainty. Every observed native CLI request in the loopback +structured-output transport check contained the complete revised system prompt. +The focused unit suite passes 36 tests. These checks establish the synthetic behavior, +not the quality of a real-suite rerun. See the +[prompt-update acceptance evidence](evidence/hermes_suite_prompt_v2_20260929.json). + ## Acceptance results, 2026-09-29 The final runs use `suite-v6-20260929`, deployed commit `299b9ed1`, including