atlas-iac/docs/evidence/hermes_suite_results_synthetic.json

884 lines
29 KiB
JSON

[
{
"size": 14,
"http_status": 200,
"request_bytes": 9847,
"client_wall_seconds": 16.947,
"exact_alias_coverage": true,
"job": {
"attempted_destinations": [
"switchyard:atlas/planning/claude",
"claude:claude-opus-4-8"
],
"compaction": false,
"compaction_signal": "CLI events and disabled compaction",
"configuration_revision": "suite-v6-20260929",
"cost_usd_estimate": 0.052175,
"created_at": 1790689083.5601273,
"duration_api_ms": 15944,
"job_id": "efa5151c8dfa4f35a52dc83244d64762",
"model": "claude-opus-4-8",
"model_usage": {
"claude-opus-4-8[1m]": {
"cacheCreationInputTokens": 0,
"cacheReadInputTokens": 0,
"canonicalModel": "claude-opus-4-8",
"contextWindow": 1000000,
"costUSD": 0.052175,
"inputTokens": 4185,
"maxOutputTokens": 64000,
"outputTokens": 1250,
"provider": "firstParty",
"webSearchRequests": 0
}
},
"result": {
"groups": [
{
"description": "Cycle thermal chamber and measure enclosure expansion with a calibrated dimensional gauge. Unique physical fixture; procedure details unknown, so no reuse with other families.",
"members": [
"CASE-0001"
],
"name": "Thermal Chamber Expansion Measurement"
},
{
"description": "Drive controllable physical watchdog pulses and capture reset-line timing with a digital capture fixture, checking deadlines across nominal/boundary/malformed inputs. Shared pulse-source and recorder machinery.",
"members": [
"CASE-0002",
"CASE-0013"
],
"name": "Watchdog Pulse Reset-Line Timing"
},
{
"description": "Parse offline watchdog static-analysis reports, count findings by severity, and compare rule IDs against a policy table. Shared report-parser and severity-policy fixtures; analysis method.",
"members": [
"CASE-0003",
"CASE-0009"
],
"name": "Static-Analysis Findings Parser"
},
{
"description": "Load configuration text fixtures through the parser and assert accepted values or diagnostic positions. Shared config-text builders and parser-call harness.",
"members": [
"CASE-0004",
"CASE-0010"
],
"name": "Configuration Parser Validation"
},
{
"description": "Drive concurrent producer/consumer tasks against a bounded queue fixture and assert depth, rejected writes, ordering, and recovery. Shared task drivers and sequence assertions.",
"members": [
"CASE-0005",
"CASE-0011"
],
"name": "Bounded Message Queue Producer/Consumer"
},
{
"description": "Submit role-scoped operations to the authorization decision function using identity fixtures and in-memory policy store; compare allow/deny decisions and audit fields. Shared policy machinery.",
"members": [
"CASE-0006",
"CASE-0012"
],
"name": "Authorization Decision Matrix"
},
{
"description": "Build encoded watchdog status frames via byte-array builder and check decoded fields, checksum status, and rejection codes. Distinct decoder-call fixture; single case.",
"members": [
"CASE-0007"
],
"name": "Watchdog Status Frame Decoder"
},
{
"description": "Record acoustic output in an anechoic fixture and verify spectral peak magnitude against a frequency-dependent threshold. Unique measurement setup; no shared machinery.",
"members": [
"CASE-0008"
],
"name": "Anechoic Acoustic Output Measurement"
},
{
"description": "Rebuild the same source twice in clean build containers and compare artifact digests after removing allowed timestamp metadata. Unique build-container machinery.",
"members": [
"CASE-0014"
],
"name": "Reproducible Build Digest Comparison"
}
]
},
"result_retention_seconds": 3600,
"routing": {
"allow_external": true,
"allowed_external_providers": [
"claude"
]
},
"selection": {
"backend": "claude-code-2.1.226",
"case_count": 14,
"cli_model": "claude-opus-4-8[1m]",
"configuration_revision": "suite-v6-20260929",
"context": 1000000,
"enabled": true,
"input_bytes": 11035,
"input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer",
"input_token_bound": 19227,
"input_token_count": null,
"model": "claude-opus-4-8",
"output": 64000,
"output_reservation_tokens": 10910,
"output_reservation_verified": false,
"overhead": 8192,
"provider": "claude",
"reasoning": "medium",
"source_sha256": "99fd35cb089e57285670f6def4339a86754316e67ce3e53ec7efa95968534ca9"
},
"status": "completed",
"temporary_files_deleted": true,
"truncation": false,
"turns": 2,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"inference_geo": "not_available",
"input_tokens": 4185,
"iterations": [],
"output_tokens": 1250,
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
},
"service_tier": "standard",
"speed": "standard"
},
"wall_seconds": 16.706
}
},
{
"size": 75,
"http_status": 200,
"request_bytes": 55984,
"client_wall_seconds": 25.755,
"exact_alias_coverage": true,
"job": {
"attempted_destinations": [
"switchyard:atlas/planning/claude",
"claude:claude-opus-4-8"
],
"compaction": false,
"compaction_signal": "CLI events and disabled compaction",
"configuration_revision": "suite-v6-20260929",
"cost_usd_estimate": 0.15412499999999998,
"created_at": 1790689100.8911846,
"duration_api_ms": 24235,
"job_id": "f0e076ffe90643779593d778920cffc7",
"model": "claude-opus-4-8",
"model_usage": {
"claude-opus-4-8[1m]": {
"cacheCreationInputTokens": 0,
"cacheReadInputTokens": 0,
"canonicalModel": "claude-opus-4-8",
"contextWindow": 1000000,
"costUSD": 0.15412499999999998,
"inputTokens": 19145,
"maxOutputTokens": 64000,
"outputTokens": 2336,
"provider": "firstParty",
"webSearchRequests": 0
}
},
"result": {
"groups": [
{
"description": "Shared byte-array builder and decoder-call fixture; assert decoded fields, checksum status, and rejection code. Same machinery across nominal, boundary, and fault-injection variants; parameterize inputs and expected results.",
"members": [
"CASE-0007",
"CASE-0013",
"CASE-0019",
"CASE-0025",
"CASE-0031",
"CASE-0037",
"CASE-0043",
"CASE-0049",
"CASE-0055",
"CASE-0061",
"CASE-0067",
"CASE-0073"
],
"name": "Watchdog Status Frame Decoder"
},
{
"description": "Shared controllable pulse source and reset-line recorder with digital capture; measure reset-line timing against a deadline. One driver/measurement harness parameterized over input variations.",
"members": [
"CASE-0002",
"CASE-0008",
"CASE-0014",
"CASE-0020",
"CASE-0026",
"CASE-0032",
"CASE-0044",
"CASE-0050",
"CASE-0056",
"CASE-0062",
"CASE-0068",
"CASE-0074"
],
"name": "Physical Watchdog Pulse Timing"
},
{
"description": "Analysis-method family sharing a static-analysis report parser and severity policy fixture; count findings by severity and compare rule IDs to a policy table. Firmware not executed.",
"members": [
"CASE-0003",
"CASE-0009",
"CASE-0015",
"CASE-0021",
"CASE-0027",
"CASE-0033",
"CASE-0039",
"CASE-0045",
"CASE-0051",
"CASE-0057",
"CASE-0063",
"CASE-0069"
],
"name": "Static-Analysis Report Parser"
},
{
"description": "Shared configuration text fixtures and parser call without network stack; assert accepted values or diagnostic positions from returned parser objects. Parameterize input documents.",
"members": [
"CASE-0004",
"CASE-0010",
"CASE-0016",
"CASE-0022",
"CASE-0028",
"CASE-0034",
"CASE-0040",
"CASE-0046",
"CASE-0052",
"CASE-0058",
"CASE-0064",
"CASE-0070"
],
"name": "Configuration Document Parser"
},
{
"description": "Shared concurrent task drivers, bounded queue fixture, and sequence-number assertions; observe queue depth, rejected writes, ordering, and recovery. Same harness parameterized over load variants.",
"members": [
"CASE-0005",
"CASE-0011",
"CASE-0017",
"CASE-0023",
"CASE-0029",
"CASE-0035",
"CASE-0041",
"CASE-0047",
"CASE-0053",
"CASE-0059",
"CASE-0065",
"CASE-0071"
],
"name": "Bounded Message Queue Producer/Consumer"
},
{
"description": "Shared identity fixtures and in-memory policy store; compare allow/deny decisions and audit-event fields to a permissions matrix. No interactive login. Parameterize role-scoped operations.",
"members": [
"CASE-0006",
"CASE-0012",
"CASE-0018",
"CASE-0024",
"CASE-0030",
"CASE-0036",
"CASE-0042",
"CASE-0048",
"CASE-0054",
"CASE-0060",
"CASE-0066",
"CASE-0072"
],
"name": "Authorization Decision Function"
},
{
"description": "Standalone thermal-cycle test measuring enclosure expansion with a calibrated gauge against dimensional tolerance. Unique fixture; procedure details unknown, so no reuse with other families.",
"members": [
"CASE-0001"
],
"name": "Thermal Chamber Expansion"
},
{
"description": "Standalone anechoic-fixture recording comparing spectral peak magnitude to a frequency-dependent threshold. Unique fixture; procedure details unknown, so kept separate.",
"members": [
"CASE-0038"
],
"name": "Acoustic Output Measurement"
},
{
"description": "Standalone analysis rebuilding source twice in clean containers and comparing artifact digests after removing allowed timestamp metadata. Unique machinery; kept separate.",
"members": [
"CASE-0075"
],
"name": "Reproducible Build Digest"
}
]
},
"result_retention_seconds": 3600,
"routing": {
"allow_external": true,
"allowed_external_providers": [
"claude"
]
},
"selection": {
"backend": "claude-code-2.1.226",
"case_count": 75,
"cli_model": "claude-opus-4-8[1m]",
"configuration_revision": "suite-v6-20260929",
"context": 1000000,
"enabled": true,
"input_bytes": 57172,
"input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer",
"input_token_bound": 65364,
"input_token_count": null,
"model": "claude-opus-4-8",
"output": 64000,
"output_reservation_tokens": 18291,
"output_reservation_verified": false,
"overhead": 8192,
"provider": "claude",
"reasoning": "medium",
"source_sha256": "3cca99c228210deee67c4f7a4674bad55283440c5662f8add1c5182409447d32"
},
"status": "completed",
"temporary_files_deleted": true,
"truncation": false,
"turns": 2,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"inference_geo": "not_available",
"input_tokens": 19145,
"iterations": [],
"output_tokens": 2336,
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
},
"service_tier": "standard",
"speed": "standard"
},
"wall_seconds": 24.922
}
},
{
"size": 363,
"http_status": 200,
"request_bytes": 273761,
"client_wall_seconds": 80.071,
"exact_alias_coverage": true,
"job": {
"attempted_destinations": [
"switchyard:atlas/planning/claude",
"claude:claude-opus-4-8"
],
"compaction": false,
"compaction_signal": "CLI events and disabled compaction",
"configuration_revision": "suite-v6-20260929",
"cost_usd_estimate": 0.6629649999999999,
"created_at": 1790689126.669432,
"duration_api_ms": 78221,
"job_id": "682e87d11aa344c3b4bd33bcab7ea431",
"model": "claude-opus-4-8",
"model_usage": {
"claude-opus-4-8[1m]": {
"cacheCreationInputTokens": 0,
"cacheReadInputTokens": 0,
"canonicalModel": "claude-opus-4-8",
"contextWindow": 1000000,
"costUSD": 0.6629649999999999,
"inputTokens": 89753,
"maxOutputTokens": 64000,
"outputTokens": 8568,
"provider": "firstParty",
"webSearchRequests": 0
}
},
"result": {
"groups": [
{
"description": "Byte-array builder feeds encoded status frames to the message decoder; assert decoded fields, checksum status, and rejection code. Case-type variants are parameters over the same fixture.",
"members": [
"CASE-0007",
"CASE-0013",
"CASE-0019",
"CASE-0025",
"CASE-0031",
"CASE-0037",
"CASE-0043",
"CASE-0049",
"CASE-0055",
"CASE-0061",
"CASE-0067",
"CASE-0073",
"CASE-0079",
"CASE-0085",
"CASE-0091",
"CASE-0097",
"CASE-0103",
"CASE-0109",
"CASE-0115",
"CASE-0121",
"CASE-0127",
"CASE-0133",
"CASE-0139",
"CASE-0145",
"CASE-0151",
"CASE-0157",
"CASE-0163",
"CASE-0169",
"CASE-0175",
"CASE-0181",
"CASE-0187",
"CASE-0193",
"CASE-0199",
"CASE-0205",
"CASE-0211",
"CASE-0217",
"CASE-0223",
"CASE-0229",
"CASE-0235",
"CASE-0241",
"CASE-0247",
"CASE-0253",
"CASE-0259",
"CASE-0265",
"CASE-0271",
"CASE-0277",
"CASE-0283",
"CASE-0289",
"CASE-0295",
"CASE-0301",
"CASE-0307",
"CASE-0313",
"CASE-0319",
"CASE-0325",
"CASE-0331",
"CASE-0337",
"CASE-0343",
"CASE-0349",
"CASE-0355",
"CASE-0361"
],
"name": "Watchdog status frame decoder harness"
},
{
"description": "Controllable pulse source stops/resumes watchdog pulses; reset-line recorder and digital capture fixture measure timing against a deadline. Shared stimulus and timing assertions across variants.",
"members": [
"CASE-0002",
"CASE-0008",
"CASE-0014",
"CASE-0020",
"CASE-0026",
"CASE-0032",
"CASE-0038",
"CASE-0044",
"CASE-0050",
"CASE-0056",
"CASE-0062",
"CASE-0068",
"CASE-0074",
"CASE-0080",
"CASE-0086",
"CASE-0092",
"CASE-0098",
"CASE-0104",
"CASE-0110",
"CASE-0116",
"CASE-0122",
"CASE-0128",
"CASE-0134",
"CASE-0140",
"CASE-0146",
"CASE-0152",
"CASE-0158",
"CASE-0164",
"CASE-0170",
"CASE-0176",
"CASE-0188",
"CASE-0194",
"CASE-0200",
"CASE-0206",
"CASE-0212",
"CASE-0218",
"CASE-0224",
"CASE-0230",
"CASE-0236",
"CASE-0242",
"CASE-0248",
"CASE-0254",
"CASE-0260",
"CASE-0266",
"CASE-0272",
"CASE-0278",
"CASE-0284",
"CASE-0290",
"CASE-0296",
"CASE-0302",
"CASE-0308",
"CASE-0314",
"CASE-0320",
"CASE-0326",
"CASE-0332",
"CASE-0338",
"CASE-0344",
"CASE-0350",
"CASE-0356",
"CASE-0362"
],
"name": "Watchdog reset-line timing rig"
},
{
"description": "Loads offline static-analysis reports and a severity policy fixture; counts findings by severity and checks each rule ID against the policy table. No firmware execution; shared parser and policy comparison.",
"members": [
"CASE-0003",
"CASE-0009",
"CASE-0015",
"CASE-0021",
"CASE-0027",
"CASE-0033",
"CASE-0039",
"CASE-0045",
"CASE-0051",
"CASE-0057",
"CASE-0063",
"CASE-0069",
"CASE-0075",
"CASE-0081",
"CASE-0087",
"CASE-0093",
"CASE-0099",
"CASE-0105",
"CASE-0111",
"CASE-0117",
"CASE-0123",
"CASE-0129",
"CASE-0135",
"CASE-0141",
"CASE-0147",
"CASE-0153",
"CASE-0159",
"CASE-0165",
"CASE-0171",
"CASE-0177",
"CASE-0183",
"CASE-0189",
"CASE-0195",
"CASE-0201",
"CASE-0207",
"CASE-0213",
"CASE-0219",
"CASE-0225",
"CASE-0231",
"CASE-0237",
"CASE-0243",
"CASE-0249",
"CASE-0255",
"CASE-0261",
"CASE-0267",
"CASE-0273",
"CASE-0279",
"CASE-0285",
"CASE-0291",
"CASE-0297",
"CASE-0303",
"CASE-0309",
"CASE-0315",
"CASE-0321",
"CASE-0327",
"CASE-0333",
"CASE-0339",
"CASE-0345",
"CASE-0351",
"CASE-0357"
],
"name": "Static-analysis report parser"
},
{
"description": "Builds configuration text fixtures and calls the parser without the network stack; asserts accepted values or diagnostic positions from returned parser objects. Shared text builders and assertions.",
"members": [
"CASE-0004",
"CASE-0010",
"CASE-0016",
"CASE-0022",
"CASE-0028",
"CASE-0034",
"CASE-0040",
"CASE-0046",
"CASE-0052",
"CASE-0058",
"CASE-0064",
"CASE-0070",
"CASE-0076",
"CASE-0082",
"CASE-0088",
"CASE-0094",
"CASE-0100",
"CASE-0106",
"CASE-0112",
"CASE-0118",
"CASE-0124",
"CASE-0130",
"CASE-0136",
"CASE-0142",
"CASE-0148",
"CASE-0154",
"CASE-0160",
"CASE-0166",
"CASE-0172",
"CASE-0178",
"CASE-0184",
"CASE-0190",
"CASE-0196",
"CASE-0202",
"CASE-0208",
"CASE-0214",
"CASE-0220",
"CASE-0226",
"CASE-0232",
"CASE-0238",
"CASE-0244",
"CASE-0250",
"CASE-0256",
"CASE-0262",
"CASE-0268",
"CASE-0274",
"CASE-0280",
"CASE-0286",
"CASE-0292",
"CASE-0298",
"CASE-0304",
"CASE-0310",
"CASE-0316",
"CASE-0322",
"CASE-0328",
"CASE-0334",
"CASE-0340",
"CASE-0346",
"CASE-0352",
"CASE-0358"
],
"name": "Configuration parser harness"
},
{
"description": "Concurrent producer/consumer task drivers push a bounded queue to its limit; assert depth, rejected writes, delivery ordering via sequence numbers, and drain recovery. Shared drivers and assertions.",
"members": [
"CASE-0005",
"CASE-0011",
"CASE-0017",
"CASE-0023",
"CASE-0029",
"CASE-0035",
"CASE-0041",
"CASE-0047",
"CASE-0053",
"CASE-0059",
"CASE-0065",
"CASE-0071",
"CASE-0077",
"CASE-0083",
"CASE-0089",
"CASE-0095",
"CASE-0101",
"CASE-0107",
"CASE-0113",
"CASE-0119",
"CASE-0125",
"CASE-0131",
"CASE-0137",
"CASE-0143",
"CASE-0149",
"CASE-0155",
"CASE-0161",
"CASE-0167",
"CASE-0173",
"CASE-0179",
"CASE-0185",
"CASE-0191",
"CASE-0197",
"CASE-0203",
"CASE-0209",
"CASE-0215",
"CASE-0221",
"CASE-0227",
"CASE-0233",
"CASE-0239",
"CASE-0245",
"CASE-0251",
"CASE-0257",
"CASE-0263",
"CASE-0269",
"CASE-0275",
"CASE-0281",
"CASE-0287",
"CASE-0293",
"CASE-0299",
"CASE-0305",
"CASE-0311",
"CASE-0317",
"CASE-0323",
"CASE-0329",
"CASE-0335",
"CASE-0341",
"CASE-0347",
"CASE-0353",
"CASE-0359"
],
"name": "Bounded message queue concurrency driver"
},
{
"description": "Identity fixtures and in-memory policy store submit role-scoped operations; compare allow/deny decisions and audit-event fields against the permissions matrix. Shared fixtures and assertions.",
"members": [
"CASE-0006",
"CASE-0012",
"CASE-0018",
"CASE-0024",
"CASE-0030",
"CASE-0036",
"CASE-0042",
"CASE-0048",
"CASE-0054",
"CASE-0060",
"CASE-0066",
"CASE-0072",
"CASE-0078",
"CASE-0084",
"CASE-0090",
"CASE-0096",
"CASE-0102",
"CASE-0108",
"CASE-0114",
"CASE-0120",
"CASE-0126",
"CASE-0132",
"CASE-0138",
"CASE-0144",
"CASE-0150",
"CASE-0156",
"CASE-0162",
"CASE-0168",
"CASE-0174",
"CASE-0180",
"CASE-0186",
"CASE-0192",
"CASE-0198",
"CASE-0204",
"CASE-0210",
"CASE-0216",
"CASE-0222",
"CASE-0228",
"CASE-0234",
"CASE-0240",
"CASE-0246",
"CASE-0252",
"CASE-0258",
"CASE-0264",
"CASE-0270",
"CASE-0276",
"CASE-0282",
"CASE-0288",
"CASE-0294",
"CASE-0300",
"CASE-0306",
"CASE-0312",
"CASE-0318",
"CASE-0324",
"CASE-0330",
"CASE-0336",
"CASE-0342",
"CASE-0348",
"CASE-0354",
"CASE-0360"
],
"name": "Authorization decision function harness"
},
{
"description": "Cycles a thermal chamber and measures enclosure expansion with a calibrated dimensional gauge. Unique physical fixture; procedure details unknown, so shares no machinery with other families.",
"members": [
"CASE-0001"
],
"name": "Thermal chamber expansion measurement"
},
{
"description": "Records acoustic output in an anechoic fixture and checks spectral peak magnitude against a frequency-dependent threshold. Unique acoustic equipment; procedure details unknown.",
"members": [
"CASE-0182"
],
"name": "Anechoic acoustic emission measurement"
},
{
"description": "Rebuilds identical source twice in clean containers and compares artifact digests after stripping allowed timestamp metadata. Distinct build-analysis machinery from all runtime harnesses.",
"members": [
"CASE-0363"
],
"name": "Reproducible build digest comparison"
}
]
},
"result_retention_seconds": 3600,
"routing": {
"allow_external": true,
"allowed_external_providers": [
"claude"
]
},
"selection": {
"backend": "claude-code-2.1.226",
"case_count": 363,
"cli_model": "claude-opus-4-8[1m]",
"configuration_revision": "suite-v6-20260929",
"context": 1000000,
"enabled": true,
"input_bytes": 274949,
"input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer",
"input_token_bound": 283141,
"input_token_count": null,
"model": "claude-opus-4-8",
"output": 64000,
"output_reservation_tokens": 53139,
"output_reservation_verified": false,
"overhead": 8192,
"provider": "claude",
"reasoning": "medium",
"source_sha256": "311c96701a3be823afd2bf2c2d682da56e5f383108001f498a8a672ab41a9037"
},
"status": "completed",
"temporary_files_deleted": true,
"truncation": false,
"turns": 2,
"usage": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"inference_geo": "not_available",
"input_tokens": 89753,
"iterations": [],
"output_tokens": 8568,
"server_tool_use": {
"web_fetch_requests": 0,
"web_search_requests": 0
},
"service_tier": "standard",
"speed": "standard"
},
"wall_seconds": 78.828
}
}
]