docs: record implementation-proximity prompt acceptance
This commit is contained in:
jenkins
parent
a7319ea03b
commit
5f58df0f03
182
docs/evidence/hermes_suite_prompt_v2_20260929.json
Normal file
182
docs/evidence/hermes_suite_prompt_v2_20260929.json
Normal file
@ -0,0 +1,182 @@
|
||||
{
|
||||
"configuration_revision": "suite-v6-20260929",
|
||||
"prompt_revision": "implementation-proximity-v2-20260929",
|
||||
"prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552",
|
||||
"deployed_commit": "a7319ea03b727a159606871cf73b8e6f29b0f757",
|
||||
"synthetic_only": true,
|
||||
"fixture": "testing/fixtures/suite_implementation_proximity.json",
|
||||
"expected_answers_sent": false,
|
||||
"quality": {
|
||||
"coverage": true,
|
||||
"families": 6,
|
||||
"pair_precision": 1.0,
|
||||
"pair_recall": 1.0,
|
||||
"false_merge_pairs": 0,
|
||||
"missed_merge_pairs": 0
|
||||
},
|
||||
"description_review": "No observed cross-family objectives. Shared JSON assertions and waveform-derived duration stay with their reusable machinery; absent interface/measurement details remain explicit uncertainty.",
|
||||
"cli_prompt_propagation": [
|
||||
{
|
||||
"cases": 14,
|
||||
"requested": "claude-opus-4-8[1m]",
|
||||
"captured": [
|
||||
{
|
||||
"path": "/v1/messages?beta=true",
|
||||
"model": "claude-opus-4-8",
|
||||
"max_tokens": 64000,
|
||||
"bytes": 15617,
|
||||
"full_grouping_system_prompt": true,
|
||||
"complete_source": true
|
||||
},
|
||||
{
|
||||
"path": "/v1/messages?beta=true",
|
||||
"model": "claude-opus-4-8",
|
||||
"max_tokens": 64000,
|
||||
"bytes": 15603,
|
||||
"full_grouping_system_prompt": true,
|
||||
"complete_source": true
|
||||
}
|
||||
],
|
||||
"returncode": 0
|
||||
}
|
||||
],
|
||||
"acceptance": {
|
||||
"client_wall_seconds": 21.059,
|
||||
"job": {
|
||||
"attempted_destinations": [
|
||||
"switchyard:atlas/planning/claude",
|
||||
"claude:claude-opus-4-8"
|
||||
],
|
||||
"compaction": false,
|
||||
"compaction_signal": "CLI events and disabled compaction",
|
||||
"configuration_revision": "suite-v6-20260929",
|
||||
"cost_usd_estimate": 0.10087499999999999,
|
||||
"created_at": 1790697742.2365413,
|
||||
"duration_api_ms": 19807,
|
||||
"job_id": "661aefd2d1f14579a264a34d64bf704b",
|
||||
"model": "claude-opus-4-8",
|
||||
"model_usage": {
|
||||
"claude-opus-4-8[1m]": {
|
||||
"cacheCreationInputTokens": 0,
|
||||
"cacheReadInputTokens": 0,
|
||||
"canonicalModel": "claude-opus-4-8",
|
||||
"contextWindow": 1000000,
|
||||
"costUSD": 0.10087499999999999,
|
||||
"inputTokens": 11010,
|
||||
"maxOutputTokens": 64000,
|
||||
"outputTokens": 1833,
|
||||
"provider": "firstParty",
|
||||
"webSearchRequests": 0
|
||||
}
|
||||
},
|
||||
"prompt_revision": "implementation-proximity-v2-20260929",
|
||||
"prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552",
|
||||
"result": {
|
||||
"groups": [
|
||||
{
|
||||
"description": "Issue mode via command/readback JSON client with reset-state fixture, collect JSON snapshot, assert acceptance flag and reported state. Variants differ in true/false acceptance, added error_code field, values\u2014input/assertion variations.",
|
||||
"members": [
|
||||
"CASE-0001",
|
||||
"CASE-0005",
|
||||
"CASE-0009",
|
||||
"CASE-0013"
|
||||
],
|
||||
"name": "JSON command/readback client mode assertions"
|
||||
},
|
||||
{
|
||||
"description": "Apply mode transition via bench controller, capture output waveform with oscilloscope adapter/probe, assert amplitude and transition ordering. One variant also derives transition duration from captured samples; same capture mechanism.",
|
||||
"members": [
|
||||
"CASE-0002",
|
||||
"CASE-0006",
|
||||
"CASE-0010",
|
||||
"CASE-0014"
|
||||
],
|
||||
"name": "Oscilloscope waveform capture mode-transition tests"
|
||||
},
|
||||
{
|
||||
"description": "Parse offline static-analysis report with shared report parser and rule-policy fixture (no target execution), count findings per rule/severity, compare to qualitative acceptance bound. Variants differ only in inputs/bounds.",
|
||||
"members": [
|
||||
"CASE-0003",
|
||||
"CASE-0007",
|
||||
"CASE-0011"
|
||||
],
|
||||
"name": "Static-analysis report parsing finding-count checks"
|
||||
},
|
||||
{
|
||||
"description": "Cycle chamber temperature and measure enclosure expansion with a calibrated dimensional gauge, asserting expansion below the generalized bound. Distinct thermal/gauge machinery independent of waveform bench; single case.",
|
||||
"members": [
|
||||
"CASE-0004"
|
||||
],
|
||||
"name": "Thermal-chamber enclosure expansion measurement"
|
||||
},
|
||||
{
|
||||
"description": "Record acoustic output in anechoic enclosure via microphone capture and spectral-analysis fixture; assert each spectral peak below a frequency-dependent bound. Unique acoustic instrumentation not shared with other families; single case.",
|
||||
"members": [
|
||||
"CASE-0008"
|
||||
],
|
||||
"name": "Acoustic spectral-peak measurement"
|
||||
},
|
||||
{
|
||||
"description": "Verify a latent-state relationship where required interface, stimulus, observation mechanism, and fixture are all omitted. Insufficient evidence to determine machinery or merge; retained as standalone uncertainty.",
|
||||
"members": [
|
||||
"CASE-0012"
|
||||
],
|
||||
"name": "Underspecified latent-state verification"
|
||||
}
|
||||
]
|
||||
},
|
||||
"result_retention_seconds": 3600,
|
||||
"routing": {
|
||||
"allow_external": true,
|
||||
"allowed_external_providers": [
|
||||
"claude"
|
||||
]
|
||||
},
|
||||
"selection": {
|
||||
"backend": "claude-code-2.1.226",
|
||||
"case_count": 14,
|
||||
"cli_model": "claude-opus-4-8[1m]",
|
||||
"configuration_revision": "suite-v6-20260929",
|
||||
"context": 1000000,
|
||||
"enabled": true,
|
||||
"input_bytes": 13568,
|
||||
"input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer",
|
||||
"input_token_bound": 21760,
|
||||
"input_token_count": null,
|
||||
"model": "claude-opus-4-8",
|
||||
"output": 64000,
|
||||
"output_reservation_tokens": 10910,
|
||||
"output_reservation_verified": false,
|
||||
"overhead": 8192,
|
||||
"prompt_revision": "implementation-proximity-v2-20260929",
|
||||
"prompt_sha256": "5002baf3f38436cbf0cdbfc769b5d16f9c13bf86d6f1e32c59d5e3498e16b552",
|
||||
"provider": "claude",
|
||||
"reasoning": "medium",
|
||||
"source_sha256": "37c6ca89c6c8fb170b104b5aec4e5aae65463e5d74fe284ab7ef1ecb55a0aac3"
|
||||
},
|
||||
"status": "completed",
|
||||
"temporary_files_deleted": true,
|
||||
"truncation": false,
|
||||
"turns": 3,
|
||||
"usage": {
|
||||
"cache_creation": {
|
||||
"ephemeral_1h_input_tokens": 0,
|
||||
"ephemeral_5m_input_tokens": 0
|
||||
},
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
"inference_geo": "not_available",
|
||||
"input_tokens": 11010,
|
||||
"iterations": [],
|
||||
"output_tokens": 1833,
|
||||
"server_tool_use": {
|
||||
"web_fetch_requests": 0,
|
||||
"web_search_requests": 0
|
||||
},
|
||||
"service_tier": "standard",
|
||||
"speed": "standard"
|
||||
},
|
||||
"wall_seconds": 20.544
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -36,6 +36,17 @@ The synthetic check fixture is
|
||||
sent, never its independent expected-family mapping. No real roster job is launched
|
||||
by this deployment or its acceptance checks.
|
||||
|
||||
Verified on the deployed endpoint: the 14-case adversarial synthetic suite completed
|
||||
in 21.059 seconds with six implementation families, exact alias coverage, and zero
|
||||
incorrect or missed merge pairs against its independent expected map. Identical
|
||||
broad descriptions did not merge different machinery. Nominal/negative-result and
|
||||
extra-assertion variants shared their existing observation code; an underspecified
|
||||
case retained its uncertainty. Every observed native CLI request in the loopback
|
||||
structured-output transport check contained the complete revised system prompt.
|
||||
The focused unit suite passes 36 tests. These checks establish the synthetic behavior,
|
||||
not the quality of a real-suite rerun. See the
|
||||
[prompt-update acceptance evidence](evidence/hermes_suite_prompt_v2_20260929.json).
|
||||
|
||||
## Acceptance results, 2026-09-29
|
||||
|
||||
The final runs use `suite-v6-20260929`, deployed commit `299b9ed1`, including
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user