302 lines
8.0 KiB
JSON
302 lines
8.0 KiB
JSON
{
|
|
"configuration_revision": "suite-v6-20260929",
|
|
"deployed_commit": "299b9ed16d79aeefb3dc5590a19dc07dc54167e7",
|
|
"date": "2026-09-29",
|
|
"client_host": "titan-jh",
|
|
"client_lan_ip": "192.168.22.8",
|
|
"destination": "192.168.22.50:443",
|
|
"tls_hostname": "worker.bstein.dev",
|
|
"proxy_bypass": true,
|
|
"laptop_test_performed": false,
|
|
"synthetic_only": true,
|
|
"results": [
|
|
{
|
|
"size": 14,
|
|
"http_status": 200,
|
|
"request_bytes": 9847,
|
|
"client_wall_seconds": 16.947,
|
|
"exact_alias_coverage": true,
|
|
"server_wall_seconds": 16.706,
|
|
"provider": "claude",
|
|
"model": "claude-opus-4-8",
|
|
"context_limit": 1000000,
|
|
"output_limit": 64000,
|
|
"input_tokens": 4185,
|
|
"output_tokens": 1250,
|
|
"cli_turns": 2,
|
|
"cost_usd_estimate": 0.052175,
|
|
"singletons": 4,
|
|
"quality": {
|
|
"coverage": true,
|
|
"families": 9,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0,
|
|
"false_merge_pairs": 0,
|
|
"missed_merge_pairs": 0
|
|
},
|
|
"description_review": "No observed cross-family objective attribution; manual review against synthetic source patterns.",
|
|
"compaction": false,
|
|
"truncation": false,
|
|
"attempted_destinations": [
|
|
"switchyard:atlas/planning/claude",
|
|
"claude:claude-opus-4-8"
|
|
],
|
|
"job_id": "efa5151c8dfa4f35a52dc83244d64762",
|
|
"provider_job_attempts": 1,
|
|
"automatic_retries_observed": 0,
|
|
"rate_limit_events_observed": 0
|
|
},
|
|
{
|
|
"size": 75,
|
|
"http_status": 200,
|
|
"request_bytes": 55984,
|
|
"client_wall_seconds": 25.755,
|
|
"exact_alias_coverage": true,
|
|
"server_wall_seconds": 24.922,
|
|
"provider": "claude",
|
|
"model": "claude-opus-4-8",
|
|
"context_limit": 1000000,
|
|
"output_limit": 64000,
|
|
"input_tokens": 19145,
|
|
"output_tokens": 2336,
|
|
"cli_turns": 2,
|
|
"cost_usd_estimate": 0.15412499999999998,
|
|
"singletons": 3,
|
|
"quality": {
|
|
"coverage": true,
|
|
"families": 9,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0,
|
|
"false_merge_pairs": 0,
|
|
"missed_merge_pairs": 0
|
|
},
|
|
"description_review": "No observed cross-family objective attribution; manual review against synthetic source patterns.",
|
|
"compaction": false,
|
|
"truncation": false,
|
|
"attempted_destinations": [
|
|
"switchyard:atlas/planning/claude",
|
|
"claude:claude-opus-4-8"
|
|
],
|
|
"job_id": "f0e076ffe90643779593d778920cffc7",
|
|
"provider_job_attempts": 1,
|
|
"automatic_retries_observed": 0,
|
|
"rate_limit_events_observed": 0
|
|
},
|
|
{
|
|
"size": 363,
|
|
"http_status": 200,
|
|
"request_bytes": 273761,
|
|
"client_wall_seconds": 80.071,
|
|
"exact_alias_coverage": true,
|
|
"server_wall_seconds": 78.828,
|
|
"provider": "claude",
|
|
"model": "claude-opus-4-8",
|
|
"context_limit": 1000000,
|
|
"output_limit": 64000,
|
|
"input_tokens": 89753,
|
|
"output_tokens": 8568,
|
|
"cli_turns": 2,
|
|
"cost_usd_estimate": 0.6629649999999999,
|
|
"singletons": 3,
|
|
"quality": {
|
|
"coverage": true,
|
|
"families": 9,
|
|
"pair_precision": 1.0,
|
|
"pair_recall": 1.0,
|
|
"false_merge_pairs": 0,
|
|
"missed_merge_pairs": 0
|
|
},
|
|
"description_review": "No observed cross-family objective attribution; manual review against synthetic source patterns.",
|
|
"compaction": false,
|
|
"truncation": false,
|
|
"attempted_destinations": [
|
|
"switchyard:atlas/planning/claude",
|
|
"claude:claude-opus-4-8"
|
|
],
|
|
"job_id": "682e87d11aa344c3b4bd33bcab7ea431",
|
|
"provider_job_attempts": 1,
|
|
"automatic_retries_observed": 0,
|
|
"rate_limit_events_observed": 0
|
|
}
|
|
],
|
|
"field_lengths": {
|
|
"description": {
|
|
"min_bytes": 48,
|
|
"median_bytes": 234,
|
|
"max_bytes": 245
|
|
},
|
|
"preconditions": {
|
|
"min_bytes": 67,
|
|
"median_bytes": 136,
|
|
"max_bytes": 143
|
|
},
|
|
"case_type": {
|
|
"min_bytes": 7,
|
|
"median_bytes": 8,
|
|
"max_bytes": 15
|
|
},
|
|
"success_criteria": {
|
|
"min_bytes": 73,
|
|
"median_bytes": 132,
|
|
"max_bytes": 135
|
|
}
|
|
},
|
|
"whole_input_mock_transport": [
|
|
{
|
|
"cases": 14,
|
|
"requested": "claude-opus-4-8[1m]",
|
|
"captured": [
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 12832,
|
|
"complete_source": true
|
|
},
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 12818,
|
|
"complete_source": true
|
|
}
|
|
],
|
|
"returncode": 0
|
|
},
|
|
{
|
|
"cases": 75,
|
|
"requested": "claude-opus-4-8[1m]",
|
|
"captured": [
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 60921,
|
|
"complete_source": true
|
|
},
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 60907,
|
|
"complete_source": true
|
|
}
|
|
],
|
|
"returncode": 0
|
|
},
|
|
{
|
|
"cases": 363,
|
|
"requested": "claude-opus-4-8[1m]",
|
|
"captured": [
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 287914,
|
|
"complete_source": true
|
|
},
|
|
{
|
|
"path": "/v1/messages?beta=true",
|
|
"model": "claude-opus-4-8",
|
|
"max_tokens": 64000,
|
|
"bytes": 287900,
|
|
"complete_source": true
|
|
}
|
|
],
|
|
"returncode": 0
|
|
}
|
|
],
|
|
"policy_checks": {
|
|
"synthetic_scope": {
|
|
"configuration_revision": "suite-v6-20260929",
|
|
"allowed_external_providers": [
|
|
"claude",
|
|
"codex"
|
|
],
|
|
"generalized_external_providers": [],
|
|
"external_scope": "exact_synthetic_fixtures"
|
|
},
|
|
"local_scope": {
|
|
"configuration_revision": "suite-v6-20260929",
|
|
"allowed_external_providers": [],
|
|
"generalized_external_providers": [],
|
|
"external_scope": "exact_synthetic_fixtures"
|
|
},
|
|
"operational_scope": {
|
|
"configuration_revision": "suite-v6-20260929",
|
|
"allowed_external_providers": [
|
|
"claude"
|
|
],
|
|
"generalized_external_providers": [
|
|
"claude"
|
|
],
|
|
"external_scope": "generalized_claude_and_exact_synthetic_fixtures"
|
|
},
|
|
"synthetic_modified_synthetic_preflight": {
|
|
"status": 403,
|
|
"error": "external_data_not_approved",
|
|
"provider": null
|
|
},
|
|
"local_modified_synthetic_preflight": {
|
|
"status": 403,
|
|
"error": "provider_forbidden",
|
|
"provider": null
|
|
},
|
|
"operational_modified_synthetic_preflight": {
|
|
"status": 200,
|
|
"error": null,
|
|
"provider": "claude"
|
|
},
|
|
"operational_codex": {
|
|
"status": 403,
|
|
"error": "provider_forbidden"
|
|
},
|
|
"operational_claude_codex": {
|
|
"status": 403,
|
|
"error": "provider_forbidden"
|
|
},
|
|
"operational_unknown": {
|
|
"status": 400,
|
|
"error": "unknown_provider"
|
|
},
|
|
"local_only_True": {
|
|
"status": 422,
|
|
"error": {
|
|
"code": "capacity_or_unsupported_backend",
|
|
"details": {
|
|
"candidates": {
|
|
"local": "capacity"
|
|
},
|
|
"input_bytes": 11069,
|
|
"output_estimate": 2718
|
|
}
|
|
}
|
|
},
|
|
"local_only_False": {
|
|
"status": 422,
|
|
"error": {
|
|
"code": "capacity_or_unsupported_backend",
|
|
"details": {
|
|
"candidates": {
|
|
"local": "capacity"
|
|
},
|
|
"input_bytes": 11069,
|
|
"output_estimate": 2718
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"oversize_curl": {
|
|
"http_status": 413,
|
|
"expect_continue": true
|
|
},
|
|
"unknowns": [
|
|
"Provider training/data-sharing/retention settings",
|
|
"Physical provider serving hardware and region",
|
|
"Remaining account quota",
|
|
"Accurate tokenizer and full 1M-context benchmark",
|
|
"Real-case grouping quality",
|
|
"Model internal attention to every input field"
|
|
]
|
|
}
|