525 lines
18 KiB
JSON
525 lines
18 KiB
JSON
{
|
|
"hosted_synthetic_14": [
|
|
{
|
|
"cost_usd_estimate": 0.42616,
|
|
"final_groups": [
|
|
{
|
|
"description": "Work-size part 1/2 of one family. Byte-array frame builder, decoder-call fixture, capture of decoded object, comparison of fields, checksum status and rejection code; no hardware",
|
|
"members": [
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10"
|
|
],
|
|
"name": "Status frame decoder harness (1/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 2/2 of one family. Byte-array frame builder, decoder-call fixture, capture of decoded object, comparison of fields, checksum status and rejection code; no hardware",
|
|
"members": [
|
|
"CASE-REC-00",
|
|
"CASE-REC-06",
|
|
"CASE-REC-12"
|
|
],
|
|
"name": "Status frame decoder harness (2/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 1/2 of one family. Controllable pulse source driver, reset-line recorder/digital capture fixture, reset timing extraction, deadline check against supplied tolerance",
|
|
"members": [
|
|
"CASE-REC-01",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-11"
|
|
],
|
|
"name": "Watchdog pulse reset timing rig (1/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 2/2 of one family. Controllable pulse source driver, reset-line recorder/digital capture fixture, reset timing extraction, deadline check against supplied tolerance",
|
|
"members": [
|
|
"CASE-REC-03",
|
|
"CASE-REC-09",
|
|
"CASE-REC-13"
|
|
],
|
|
"name": "Watchdog pulse reset timing rig (2/2)"
|
|
}
|
|
],
|
|
"mode": "direct",
|
|
"natural_groups": [
|
|
{
|
|
"common_work": "Byte-array frame builder, decoder-call fixture, capture of decoded object, comparison of fields, checksum status and rejection code; no hardware",
|
|
"description": "Offline tests: a byte-array builder feeds encoded watchdog status frames to a decoder-call fixture. Checks decoded fields, checksum status and rejection code. Nominal and fault cases differ only in bytes and expected results.",
|
|
"evidence": [
|
|
{
|
|
"alias": "CASE-REC-00",
|
|
"field": "success_criteria",
|
|
"quote": "Capture the decoded object and compare fields, checksum status, and rejection code"
|
|
},
|
|
{
|
|
"alias": "CASE-REC-02",
|
|
"field": "preconditions",
|
|
"quote": "no live hardware is required"
|
|
}
|
|
],
|
|
"members": [
|
|
"CASE-REC-00",
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-06",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10",
|
|
"CASE-REC-12"
|
|
],
|
|
"name": "Status frame decoder harness",
|
|
"rationale": "All members have the same preconditions, the same stimulus path and the same success criteria. Fault-injection cases change only the frame bytes and the expected checksum or rejection outcome. They reuse the fixture and the assertion structure.",
|
|
"uncertainty": "Frame contents, the faults to inject and the expected rejection codes are not stated. Complex corruption patterns may need extra builder helpers.",
|
|
"variation_sets": [
|
|
[
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10"
|
|
],
|
|
[
|
|
"CASE-REC-00",
|
|
"CASE-REC-06",
|
|
"CASE-REC-12"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"common_work": "Controllable pulse source driver, reset-line recorder/digital capture fixture, reset timing extraction, deadline check against supplied tolerance",
|
|
"description": "Hardware tests: a controllable pulse source stops and resumes watchdog pulses at the controller input. A digital capture fixture records reset-line timing against the supplied deadline. Nominal and fault cases differ in stimulus only.",
|
|
"evidence": [
|
|
{
|
|
"alias": "CASE-REC-01",
|
|
"field": "success_criteria",
|
|
"quote": "Measure reset-line timing with a digital capture fixture and check the deadline"
|
|
},
|
|
{
|
|
"alias": "CASE-REC-03",
|
|
"field": "preconditions",
|
|
"quote": "Use a controllable pulse source and reset-line recorder"
|
|
}
|
|
],
|
|
"members": [
|
|
"CASE-REC-01",
|
|
"CASE-REC-03",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-09",
|
|
"CASE-REC-11",
|
|
"CASE-REC-13"
|
|
],
|
|
"name": "Watchdog pulse reset timing rig",
|
|
"rationale": "All members use the same physical rig, the same stop/resume pulse sequence and the same timing measurement and deadline assertion. The fault-injection label adds no new equipment or new way of observing. This is separate from the offline decoder work.",
|
|
"uncertainty": "Deadline and tolerance values are left out. The records don't say how fault cases change the pulse stimulus, for example stop duration or irregular pulses. Such changes might need extra pulse-source control.",
|
|
"variation_sets": [
|
|
[
|
|
"CASE-REC-01",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-11",
|
|
"CASE-REC-13"
|
|
],
|
|
[
|
|
"CASE-REC-03",
|
|
"CASE-REC-09"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"passes": [
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 11122,
|
|
"max_turns": 4,
|
|
"stage": "proposal_a",
|
|
"wall_seconds": 5.724
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 11122,
|
|
"max_turns": 4,
|
|
"stage": "proposal_b",
|
|
"wall_seconds": 9.437
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 16696,
|
|
"max_turns": 4,
|
|
"stage": "reconciliation",
|
|
"wall_seconds": 23.197
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 20816,
|
|
"max_turns": 4,
|
|
"stage": "large_family_review",
|
|
"wall_seconds": 26.503
|
|
},
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 25569,
|
|
"max_turns": 4,
|
|
"stage": "decision_audit",
|
|
"wall_seconds": 14.143
|
|
}
|
|
],
|
|
"quality": {
|
|
"coverage": true,
|
|
"false_merge_pairs": 0,
|
|
"families": 2,
|
|
"missed_merge_pairs": 0,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0
|
|
},
|
|
"seconds": 79.00923439487815,
|
|
"status": "completed",
|
|
"usage": {
|
|
"cache_creation": {
|
|
"ephemeral_1h_input_tokens": 0,
|
|
"ephemeral_5m_input_tokens": 0
|
|
},
|
|
"cache_creation_input_tokens": 0,
|
|
"cache_read_input_tokens": 0,
|
|
"input_tokens": 60130,
|
|
"output_tokens": 9282,
|
|
"output_tokens_details": {
|
|
"thinking_tokens": 604
|
|
},
|
|
"server_tool_use": {
|
|
"web_fetch_requests": 0,
|
|
"web_search_requests": 0
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cost_usd_estimate": 0.801868,
|
|
"final_groups": [
|
|
{
|
|
"description": "Work-size part 1/2 of one family. Byte-array frame builder, hardware-free decoder-call fixture, capture of decoded object, and field/checksum/rejection-code comparison",
|
|
"members": [
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10"
|
|
],
|
|
"name": "Status Frame Decoder Byte Harness (1/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 2/2 of one family. Byte-array frame builder, hardware-free decoder-call fixture, capture of decoded object, and field/checksum/rejection-code comparison",
|
|
"members": [
|
|
"CASE-REC-00",
|
|
"CASE-REC-06",
|
|
"CASE-REC-12"
|
|
],
|
|
"name": "Status Frame Decoder Byte Harness (2/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 1/2 of one family. Controllable pulse source at controller input, digital capture of the reset line, and a deadline check within the supplied timing tolerance",
|
|
"members": [
|
|
"CASE-REC-01",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-11"
|
|
],
|
|
"name": "Watchdog Pulse Reset-Line Timing Capture (1/2)"
|
|
},
|
|
{
|
|
"description": "Work-size part 2/2 of one family. Controllable pulse source at controller input, digital capture of the reset line, and a deadline check within the supplied timing tolerance",
|
|
"members": [
|
|
"CASE-REC-03",
|
|
"CASE-REC-09",
|
|
"CASE-REC-13"
|
|
],
|
|
"name": "Watchdog Pulse Reset-Line Timing Capture (2/2)"
|
|
}
|
|
],
|
|
"mode": "hierarchical",
|
|
"natural_groups": [
|
|
{
|
|
"common_work": "Byte-array frame builder, hardware-free decoder-call fixture, capture of decoded object, and field/checksum/rejection-code comparison",
|
|
"description": "Software-only tests: a byte-array builder encodes watchdog status frames and the decoder is called. Decoded fields, checksum status and rejection code are asserted. Frames are nominal or fault-injected, but the faults used are not stated.",
|
|
"evidence": [
|
|
{
|
|
"alias": "CASE-REC-00",
|
|
"field": "success_criteria",
|
|
"quote": "Capture the decoded object and compare fields, checksum status, and rejection code"
|
|
},
|
|
{
|
|
"alias": "CASE-REC-02",
|
|
"field": "preconditions",
|
|
"quote": "Use a byte-array builder and a decoder-call fixture; no live hardware is required"
|
|
}
|
|
],
|
|
"members": [
|
|
"CASE-REC-00",
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-06",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10",
|
|
"CASE-REC-12"
|
|
],
|
|
"name": "Status Frame Decoder Byte Harness",
|
|
"rationale": "All members have the same preconditions and success_criteria. Nominal and fault-injected cases change only the frame bytes and the expected decode or rejection result. Once the builder and comparison helper exist, each extra case is a cheap input and assertion variation.",
|
|
"uncertainty": "The source does not state which corruptions the fault-injection frames use or which rejection codes they expect. Identical text may still hide different frame contents.",
|
|
"variation_sets": [
|
|
[
|
|
"CASE-REC-02",
|
|
"CASE-REC-04",
|
|
"CASE-REC-08",
|
|
"CASE-REC-10"
|
|
],
|
|
[
|
|
"CASE-REC-00",
|
|
"CASE-REC-06",
|
|
"CASE-REC-12"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"common_work": "Controllable pulse source at controller input, digital capture of the reset line, and a deadline check within the supplied timing tolerance",
|
|
"description": "Hardware tests: a pulse source stops and resumes watchdog pulses at the controller input. A digital capture records reset-line timing for a deadline check within tolerance. Deadlines and fault pulse patterns are not stated.",
|
|
"evidence": [
|
|
{
|
|
"alias": "CASE-REC-01",
|
|
"field": "success_criteria",
|
|
"quote": "Measure reset-line timing with a digital capture fixture and check the deadline"
|
|
},
|
|
{
|
|
"alias": "CASE-REC-03",
|
|
"field": "preconditions",
|
|
"quote": "Use a controllable pulse source and reset-line recorder; timing tolerance is supplied"
|
|
}
|
|
],
|
|
"members": [
|
|
"CASE-REC-01",
|
|
"CASE-REC-03",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-09",
|
|
"CASE-REC-11",
|
|
"CASE-REC-13"
|
|
],
|
|
"name": "Watchdog Pulse Reset-Line Timing Capture",
|
|
"rationale": "All members share the same physical stimulus equipment and digital-capture timing measurement. This is materially different from the software decoder harness. Nominal and fault cases differ only in pulse pattern and expected timing, using the same equipment and workflow.",
|
|
"uncertainty": "Deadline values, tolerances and the fault-injection pulse patterns are not stated. Fault cases might need stimulus control beyond simply stopping and resuming pulses.",
|
|
"variation_sets": [
|
|
[
|
|
"CASE-REC-01",
|
|
"CASE-REC-05",
|
|
"CASE-REC-07",
|
|
"CASE-REC-11",
|
|
"CASE-REC-13"
|
|
],
|
|
[
|
|
"CASE-REC-03",
|
|
"CASE-REC-09"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"passes": [
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 9394,
|
|
"max_turns": 4,
|
|
"stage": "implementation_profiles",
|
|
"wall_seconds": 10.335
|
|
},
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 9386,
|
|
"max_turns": 4,
|
|
"stage": "implementation_profiles",
|
|
"wall_seconds": 10.134
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 9386,
|
|
"max_turns": 4,
|
|
"stage": "implementation_profiles",
|
|
"wall_seconds": 19.774
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 7049,
|
|
"max_turns": 4,
|
|
"stage": "implementation_profiles",
|
|
"wall_seconds": 12.544
|
|
},
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 17434,
|
|
"max_turns": 4,
|
|
"stage": "profile_proposal_a",
|
|
"wall_seconds": 5.923
|
|
},
|
|
{
|
|
"cli_turns": 2,
|
|
"input_bytes": 17434,
|
|
"max_turns": 4,
|
|
"stage": "profile_proposal_b",
|
|
"wall_seconds": 5.723
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 18258,
|
|
"max_turns": 4,
|
|
"stage": "profile_reconciliation",
|
|
"wall_seconds": 9.535
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 16167,
|
|
"max_turns": 4,
|
|
"stage": "source_review",
|
|
"wall_seconds": 21.383
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 20937,
|
|
"max_turns": 4,
|
|
"stage": "large_family_review",
|
|
"wall_seconds": 26.395
|
|
},
|
|
{
|
|
"cli_turns": 3,
|
|
"input_bytes": 25745,
|
|
"max_turns": 4,
|
|
"stage": "decision_audit",
|
|
"wall_seconds": 26.992
|
|
}
|
|
],
|
|
"quality": {
|
|
"coverage": true,
|
|
"false_merge_pairs": 0,
|
|
"families": 2,
|
|
"missed_merge_pairs": 0,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0
|
|
},
|
|
"seconds": 148.75216588703915,
|
|
"status": "completed",
|
|
"usage": {
|
|
"cache_creation": {
|
|
"ephemeral_1h_input_tokens": 0,
|
|
"ephemeral_5m_input_tokens": 0
|
|
},
|
|
"cache_creation_input_tokens": 0,
|
|
"cache_read_input_tokens": 0,
|
|
"input_tokens": 111617,
|
|
"output_tokens": 17770,
|
|
"output_tokens_details": {
|
|
"thinking_tokens": 635
|
|
},
|
|
"server_tool_use": {
|
|
"web_fetch_requests": 0,
|
|
"web_search_requests": 0
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"mocked_synthetic_363": [
|
|
{
|
|
"mode": "direct",
|
|
"backend": "mocked-independent-fixture-oracle",
|
|
"elapsed_seconds": 6.596287971013226,
|
|
"source_size_metadata": {
|
|
"source_bytes": 410322,
|
|
"description": {
|
|
"min": 113,
|
|
"max": 650,
|
|
"median": 133,
|
|
"p95": 641
|
|
},
|
|
"preconditions": {
|
|
"min": 108,
|
|
"max": 694,
|
|
"median": 131,
|
|
"p95": 682
|
|
},
|
|
"case_type": {
|
|
"min": 7,
|
|
"max": 15,
|
|
"median": 8,
|
|
"p95": 15
|
|
},
|
|
"success_criteria": {
|
|
"min": 149,
|
|
"max": 1009,
|
|
"median": 164,
|
|
"p95": 1002
|
|
}
|
|
},
|
|
"request_bytes": 410474,
|
|
"result_bytes": 175646,
|
|
"model_calls": 5,
|
|
"natural_families": 45,
|
|
"final_tasks": 90,
|
|
"max_group_size": 5,
|
|
"fixture_assignment_validation": {
|
|
"coverage": true,
|
|
"families": 45,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0,
|
|
"false_merge_pairs": 0,
|
|
"missed_merge_pairs": 0
|
|
},
|
|
"measured_provider_usage": null,
|
|
"measured_provider_cost": null
|
|
},
|
|
{
|
|
"mode": "hierarchical",
|
|
"backend": "mocked-independent-fixture-oracle",
|
|
"elapsed_seconds": 6.578743355930783,
|
|
"source_size_metadata": {
|
|
"source_bytes": 410322,
|
|
"description": {
|
|
"min": 113,
|
|
"max": 650,
|
|
"median": 133,
|
|
"p95": 641
|
|
},
|
|
"preconditions": {
|
|
"min": 108,
|
|
"max": 694,
|
|
"median": 131,
|
|
"p95": 682
|
|
},
|
|
"case_type": {
|
|
"min": 7,
|
|
"max": 15,
|
|
"median": 8,
|
|
"p95": 15
|
|
},
|
|
"success_criteria": {
|
|
"min": 149,
|
|
"max": 1009,
|
|
"median": 164,
|
|
"p95": 1002
|
|
}
|
|
},
|
|
"request_bytes": 410474,
|
|
"result_bytes": 224822,
|
|
"model_calls": 26,
|
|
"natural_families": 45,
|
|
"final_tasks": 90,
|
|
"max_group_size": 5,
|
|
"fixture_assignment_validation": {
|
|
"coverage": true,
|
|
"families": 45,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0,
|
|
"false_merge_pairs": 0,
|
|
"missed_merge_pairs": 0
|
|
},
|
|
"measured_provider_usage": null,
|
|
"measured_provider_cost": null
|
|
}
|
|
],
|
|
"unit_tests": 200,
|
|
"real_suite_rerun": false,
|
|
"large_hosted_quality_verified": false
|
|
}
|