diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index aa06ac1c..0b22498f 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5198,6 +5198,7 @@ def build_ai_dashboard(): *, unit="percent", authenticated_expr=None, + description=None, ): thresholds = remaining_thresholds if unit != "percent": @@ -5210,6 +5211,12 @@ def build_ai_dashboard(): } if authenticated_expr: expr = f"({expr}) or on() (({authenticated_expr}) * 0 - 2)" + quota_description = ( + "A last-good snapshot remains visible for up to 20 minutes through " + "a transient fetch miss. Not exposed means access is healthy but the " + "credential cannot read this quota; unavailable means access is " + "unhealthy or the snapshot is stale." + ) panel = stat_panel( panel_id, title, @@ -5219,7 +5226,11 @@ def build_ai_dashboard(): decimals=1, thresholds=thresholds, instant=True, - description="Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale.", + description=( + f"{description} {quota_description}" + if description + else f"Live first-party CLI account telemetry. {quota_description}" + ), ) panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping return panel @@ -5235,59 +5246,166 @@ def build_ai_dashboard(): openai_snapshot = snapshot_fresh("openai") anthropic_snapshot = snapshot_fresh("anthropic") + openai_authenticated = 'atlas_ai_provider_authenticated{provider="openai"} == 1' anthropic_authenticated = ( 'atlas_ai_provider_authenticated{provider="anthropic"} == 1' ) quota = "atlas_ai_quota_remaining_percent" reset = "atlas_ai_quota_reset_timestamp_seconds" - claude_models = 'model=~"route/claude/.+"' - def claude_routed_tokens(window): - """Count the disjoint Claude token classes Switchyard observed.""" + def provider_quota(metric, provider, window): + """Aggregate provider quota buckets into one conservative KPI.""" + return ( + f'min by (provider) ({metric}{{provider="{provider}",' + f'window="{window}"}})' + ) + + def provider_routed_tokens(provider, window): + """Count equivalent routed token classes for one hosted provider.""" + provider_models = f'model=~"(route|worker)/{provider}/.+"' metrics = ( "switchyard_prompt_tokens_total", "switchyard_cached_tokens_total", "switchyard_cache_creation_tokens_total", + "switchyard_reasoning_tokens_total", "switchyard_completion_tokens_total", ) terms = [ - f'(sum(increase({metric}{{{claude_models}}}[{window}])) or on() vector(0))' + f'(sum(increase({metric}{{{provider_models}}}[{window}])) ' + "or on() vector(0))" for metric in metrics ] return " + ".join(terms) + remaining_description = ( + "Lowest remaining percentage across the provider's active first-party " + "quota buckets for this window. This conservative value shows the limit " + "most likely to constrain new work." + ) + reset_description = ( + "Time until the earliest active first-party quota reset for this provider " + "and window." + ) + routed_tokens_description = ( + "Prompt, cache-read, cache-creation, reasoning, and completion tokens " + "observed by Switchyard for this provider, including interactive and " + "worker routes." + ) + panels = [ quota_stat( 1, - "Codex Weekly Remaining", - f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_snapshot})', + "Codex 5h Remaining", + f'{provider_quota(quota, "openai", "five_hour")} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 0, "y": 0}, + authenticated_expr=openai_authenticated, + description=remaining_description, ), quota_stat( 2, - "Codex Spark Weekly Remaining", - f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_snapshot})', + "Codex 5h Reset In", + f'clamp_min({provider_quota(reset, "openai", "five_hour")} - time(), 0) and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 4, "y": 0}, + unit="s", + authenticated_expr=openai_authenticated, + description=reset_description, ), quota_stat( 3, - "Claude 5h Remaining", - f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_snapshot})', + "Codex 7d Remaining", + f'{provider_quota(quota, "openai", "seven_day")} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 8, "y": 0}, - authenticated_expr=anthropic_authenticated, + authenticated_expr=openai_authenticated, + description=remaining_description, ), quota_stat( 4, - "Claude 7d Remaining", - f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_snapshot})', + "Codex 7d Reset In", + f'clamp_min({provider_quota(reset, "openai", "seven_day")} - time(), 0) and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 12, "y": 0}, - authenticated_expr=anthropic_authenticated, + unit="s", + authenticated_expr=openai_authenticated, + description=reset_description, ), stat_panel( 5, + "Codex Routed Tokens (24h)", + provider_routed_tokens("codex", "24h"), + {"h": 4, "w": 4, "x": 16, "y": 0}, + unit="short", + decimals=0, + instant=True, + description=routed_tokens_description, + ), + stat_panel( + 6, + "Codex Routed Tokens (7d)", + provider_routed_tokens("codex", "7d"), + {"h": 4, "w": 4, "x": 20, "y": 0}, + unit="short", + decimals=0, + instant=True, + description=routed_tokens_description, + ), + quota_stat( + 7, + "Claude 5h Remaining", + f'{provider_quota(quota, "anthropic", "five_hour")} and on(provider) ({anthropic_snapshot})', + {"h": 4, "w": 4, "x": 0, "y": 4}, + authenticated_expr=anthropic_authenticated, + description=remaining_description, + ), + quota_stat( + 8, + "Claude 5h Reset In", + f'clamp_min({provider_quota(reset, "anthropic", "five_hour")} - time(), 0) and on(provider) ({anthropic_snapshot})', + {"h": 4, "w": 4, "x": 4, "y": 4}, + unit="s", + authenticated_expr=anthropic_authenticated, + description=reset_description, + ), + quota_stat( + 9, + "Claude 7d Remaining", + f'{provider_quota(quota, "anthropic", "seven_day")} and on(provider) ({anthropic_snapshot})', + {"h": 4, "w": 4, "x": 8, "y": 4}, + authenticated_expr=anthropic_authenticated, + description=remaining_description, + ), + quota_stat( + 10, + "Claude 7d Reset In", + f'clamp_min({provider_quota(reset, "anthropic", "seven_day")} - time(), 0) and on(provider) ({anthropic_snapshot})', + {"h": 4, "w": 4, "x": 12, "y": 4}, + unit="s", + authenticated_expr=anthropic_authenticated, + description=reset_description, + ), + stat_panel( + 11, + "Claude Routed Tokens (24h)", + provider_routed_tokens("claude", "24h"), + {"h": 4, "w": 4, "x": 16, "y": 4}, + unit="short", + decimals=0, + instant=True, + description=routed_tokens_description, + ), + stat_panel( + 12, + "Claude Routed Tokens (7d)", + provider_routed_tokens("claude", "7d"), + {"h": 4, "w": 4, "x": 20, "y": 4}, + unit="short", + decimals=0, + instant=True, + description=routed_tokens_description, + ), + stat_panel( + 13, "Provider Access Healthy", "sum(atlas_ai_provider_authenticated) or on() vector(0)", - {"h": 4, "w": 4, "x": 16, "y": 0}, + {"h": 4, "w": 3, "x": 0, "y": 8}, instant=True, thresholds={ "mode": "absolute", @@ -5301,10 +5419,27 @@ def build_ai_dashboard(): description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.", ), stat_panel( - 6, + 14, + "Quota Fetch Healthy", + "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + {"h": 4, "w": 3, "x": 3, "y": 8}, + instant=True, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "red", "value": None}, + {"color": "yellow", "value": 1}, + {"color": "green", "value": 2}, + ], + }, + value_suffix=" / 2", + description="Provider quota collectors whose latest refresh completed successfully.", + ), + stat_panel( + 15, "Oldest Quota Sample", "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", - {"h": 4, "w": 4, "x": 20, "y": 0}, + {"h": 4, "w": 3, "x": 6, "y": 8}, unit="s", instant=True, thresholds={ @@ -5318,108 +5453,21 @@ def build_ai_dashboard(): }, description="Age of the stalest successful provider quota snapshot.", ), - quota_stat( - 7, - "Codex Weekly Reset In", - f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_snapshot})', - {"h": 4, "w": 4, "x": 0, "y": 4}, - unit="s", - ), stat_panel( - 8, - "Claude Routed Tokens (24h)", - claude_routed_tokens("24h"), - {"h": 4, "w": 4, "x": 4, "y": 4}, - unit="short", - decimals=0, - instant=True, - description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance.", - ), - stat_panel( - 9, - "Claude Routed Tokens (7d)", - claude_routed_tokens("7d"), - {"h": 4, "w": 4, "x": 8, "y": 4}, - unit="short", - decimals=0, - instant=True, - description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope.", - ), - quota_stat( - 10, - "Codex Tokens (Latest Day)", - f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_snapshot})', - {"h": 4, "w": 4, "x": 12, "y": 4}, - unit="short", - ), - quota_stat( - 11, - "Codex Tokens (7d)", - f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_snapshot})', - {"h": 4, "w": 4, "x": 16, "y": 4}, - unit="short", - ), - stat_panel( - 12, + 16, "Switchyard Requests", "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)", - {"h": 4, "w": 4, "x": 20, "y": 4}, + {"h": 4, "w": 3, "x": 9, "y": 8}, unit="short", decimals=0, instant=True, description="Hosted model requests observed by Switchyard in the selected dashboard range.", ), - timeseries_panel( - 13, - "Model Selection Rate", - "sum by (selected_model) (rate(switchyard_decisions_total[5m]))", - {"h": 8, "w": 12, "x": 0, "y": 8}, - unit="reqps", - legend="{{selected_model}}", - legend_placement="right", - description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.", - ), - bargauge_panel( - 14, - "Provider Selections (Range)", - 'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))', - {"h": 8, "w": 12, "x": 12, "y": 8}, - unit="short", - legend="{{provider}}", - instant=True, - include_color=False, - description="Switchyard selections grouped by provider over the selected dashboard range.", - ), - timeseries_panel( - 15, - "Token Throughput", - None, - {"h": 8, "w": 12, "x": 0, "y": 16}, - unit="tps", - targets=[ - {"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"}, - {"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"}, - {"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"}, - {"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"}, - {"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"}, - ], - description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.", - ), - timeseries_panel( - 16, - "Model Call p95 Latency", - "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))", - {"h": 8, "w": 12, "x": 12, "y": 16}, - unit="ms", - legend="{{model}}", - legend_placement="right", - description="95th percentile upstream latency for each selected model route.", - ), stat_panel( 17, "Prompt Cache Share", "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", - {"h": 4, "w": 6, "x": 0, "y": 24}, + {"h": 4, "w": 3, "x": 12, "y": 8}, unit="percent", decimals=1, description="Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.", @@ -5428,7 +5476,7 @@ def build_ai_dashboard(): 18, "Client Success Rate", '100 * sum(increase(switchyard_client_responses_total{outcome="success"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)', - {"h": 4, "w": 6, "x": 6, "y": 24}, + {"h": 4, "w": 3, "x": 15, "y": 8}, unit="percent", decimals=1, thresholds={ @@ -5443,9 +5491,9 @@ def build_ai_dashboard(): ), stat_panel( 19, - "Classifier Fail-Open (Range)", + "Classifier Fail-Open", "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", - {"h": 4, "w": 6, "x": 12, "y": 24}, + {"h": 4, "w": 3, "x": 18, "y": 8}, decimals=0, thresholds={ "mode": "absolute", @@ -5459,9 +5507,9 @@ def build_ai_dashboard(): ), stat_panel( 20, - "Upstream Errors (Range)", + "Upstream Errors", "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", - {"h": 4, "w": 6, "x": 18, "y": 24}, + {"h": 4, "w": 3, "x": 21, "y": 8}, decimals=0, thresholds={ "mode": "absolute", @@ -5475,6 +5523,52 @@ def build_ai_dashboard(): ), timeseries_panel( 21, + "Model Selection Rate", + "sum by (selected_model) (rate(switchyard_decisions_total[5m]))", + {"h": 8, "w": 12, "x": 0, "y": 12}, + unit="reqps", + legend="{{selected_model}}", + legend_placement="right", + description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.", + ), + bargauge_panel( + 22, + "Provider Selections (Range)", + 'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))', + {"h": 8, "w": 12, "x": 12, "y": 12}, + unit="short", + legend="{{provider}}", + instant=True, + include_color=False, + description="Switchyard selections grouped by provider over the selected dashboard range.", + ), + timeseries_panel( + 23, + "Token Throughput", + None, + {"h": 8, "w": 12, "x": 0, "y": 20}, + unit="tps", + targets=[ + {"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"}, + {"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"}, + {"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"}, + {"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"}, + {"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"}, + ], + description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.", + ), + timeseries_panel( + 24, + "Model Call p95 Latency", + "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))", + {"h": 8, "w": 12, "x": 12, "y": 20}, + unit="ms", + legend="{{model}}", + legend_placement="right", + description="95th percentile upstream latency for each selected model route.", + ), + timeseries_panel( + 25, "Local Classifier Calls", 'sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~"qwen.*"}[5m]))', {"h": 8, "w": 8, "x": 0, "y": 28}, @@ -5483,7 +5577,7 @@ def build_ai_dashboard(): description="Local Qwen routing-classifier activity, split by successful and failed calls.", ), timeseries_panel( - 22, + 26, "Routing Overhead p95", "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))", {"h": 8, "w": 8, "x": 8, "y": 28}, @@ -5492,7 +5586,7 @@ def build_ai_dashboard(): description="95th percentile time Switchyard spends selecting a model before the upstream call.", ), bargauge_panel( - 23, + 27, "Traffic Lanes (Range)", 'sum by (lane) (label_replace(increase(switchyard_requests_total{model=~"(route|worker)/.*"}[$__range]), "lane", "$1", "model", "^(route|worker)/.*"))', {"h": 8, "w": 8, "x": 16, "y": 28}, @@ -5503,13 +5597,13 @@ def build_ai_dashboard(): description="Request volume split between interactive route traffic and durable worker traffic.", ), text_panel( - 24, + 28, "Cost Semantics", "Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage.", {"h": 8, "w": 12, "x": 0, "y": 36}, ), timeseries_panel( - 25, + 29, "Hermes Workload CPU (Attribution Proxy)", 'sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace="hermes",pod=~"hermes-(agent|chat-tenant|switchyard|model-gate).*",container!="",image!=""}[5m]))', {"h": 8, "w": 12, "x": 12, "y": 36}, diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index 75be5652..d25d81c3 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -7,7 +7,7 @@ { "id": 1, "type": "stat", - "title": "Codex Weekly Remaining", + "title": "Codex 5h Remaining", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -91,12 +91,12 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 2, "type": "stat", - "title": "Codex Spark Weekly Remaining", + "title": "Codex 5h Reset In", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -109,7 +109,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -142,24 +142,12 @@ "value": null }, { - "color": "dark-red", + "color": "dark-blue", "value": 0 - }, - { - "color": "dark-orange", - "value": 15 - }, - { - "color": "dark-yellow", - "value": 30 - }, - { - "color": "dark-green", - "value": 50 } ] }, - "unit": "percent", + "unit": "s", "custom": { "displayMode": "auto" }, @@ -180,12 +168,12 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 3, "type": "stat", - "title": "Claude 5h Remaining", + "title": "Codex 7d Remaining", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -198,7 +186,7 @@ }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -269,12 +257,12 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 4, "type": "stat", - "title": "Claude 7d Remaining", + "title": "Codex 7d Reset In", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -287,7 +275,210 @@ }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 5, + "type": "stat", + "title": "Codex Routed Tokens (24h)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 6, + "type": "stat", + "title": "Codex Routed Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 7, + "type": "stat", + "title": "Claude 5h Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -358,12 +549,255 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { - "id": 5, + "id": 8, "type": "stat", - "title": "Provider Access Healthy", + "title": "Claude 5h Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "targets": [ + { + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 9, + "type": "stat", + "title": "Claude 7d Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "targets": [ + { + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 10, + "type": "stat", + "title": "Claude 7d Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 11, + "type": "stat", + "title": "Claude Routed Tokens (24h)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -372,7 +806,133 @@ "h": 4, "w": 4, "x": 16, - "y": 0 + "y": 4 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 12, + "type": "stat", + "title": "Claude Routed Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 13, + "type": "stat", + "title": "Provider Access Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 8 }, "targets": [ { @@ -428,7 +988,74 @@ "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." }, { - "id": 6, + "id": 14, + "type": "stat", + "title": "Quota Fetch Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 8 + }, + "targets": [ + { + "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 2 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto", + "valueSuffix": " / 2" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Provider quota collectors whose latest refresh completed successfully." + }, + { + "id": 15, "type": "stat", "title": "Oldest Quota Sample", "datasource": { @@ -437,9 +1064,9 @@ }, "gridPos": { "h": 4, - "w": 4, - "x": 20, - "y": 0 + "w": 3, + "x": 6, + "y": 8 }, "targets": [ { @@ -498,364 +1125,7 @@ "description": "Age of the stalest successful provider quota snapshot." }, { - "id": 7, - "type": "stat", - "title": "Codex Weekly Reset In", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 4 - }, - "targets": [ - { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "s", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 8, - "type": "stat", - "title": "Claude Routed Tokens (24h)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 4 - }, - "targets": [ - { - "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance." - }, - { - "id": 9, - "type": "stat", - "title": "Claude Routed Tokens (7d)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 4 - }, - "targets": [ - { - "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope." - }, - { - "id": 10, - "type": "stat", - "title": "Codex Tokens (Latest Day)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 4 - }, - "targets": [ - { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 11, - "type": "stat", - "title": "Codex Tokens (7d)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 4 - }, - "targets": [ - { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 12, + "id": 16, "type": "stat", "title": "Switchyard Requests", "datasource": { @@ -864,9 +1134,9 @@ }, "gridPos": { "h": 4, - "w": 4, - "x": 20, - "y": 4 + "w": 3, + "x": 9, + "y": 8 }, "targets": [ { @@ -918,7 +1188,267 @@ "description": "Hosted model requests observed by Switchyard in the selected dashboard range." }, { - "id": 13, + "id": 17, + "type": "stat", + "title": "Prompt Cache Share", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." + }, + { + "id": 18, + "type": "stat", + "title": "Client Success Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 8 + }, + "targets": [ + { + "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 99 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful client-facing Switchyard responses in the selected range." + }, + { + "id": 19, + "type": "stat", + "title": "Classifier Fail-Open", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 8 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Local classifier failures that safely fell back to the conservative hosted route." + }, + { + "id": 20, + "type": "stat", + "title": "Upstream Errors", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 8 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model attempts that returned errors in the selected range." + }, + { + "id": 21, "type": "timeseries", "title": "Model Selection Rate", "datasource": { @@ -929,7 +1459,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "targets": [ { @@ -956,7 +1486,7 @@ "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort." }, { - "id": 14, + "id": 22, "type": "bargauge", "title": "Provider Selections (Range)", "datasource": { @@ -967,7 +1497,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "targets": [ { @@ -1031,7 +1561,7 @@ ] }, { - "id": 15, + "id": 23, "type": "timeseries", "title": "Token Throughput", "datasource": { @@ -1042,7 +1572,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "targets": [ { @@ -1089,7 +1619,7 @@ "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls." }, { - "id": 16, + "id": 24, "type": "timeseries", "title": "Model Call p95 Latency", "datasource": { @@ -1100,7 +1630,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "targets": [ { @@ -1127,267 +1657,7 @@ "description": "95th percentile upstream latency for each selected model route." }, { - "id": 17, - "type": "stat", - "title": "Prompt Cache Share", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 24 - }, - "targets": [ - { - "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "percent", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." - }, - { - "id": 18, - "type": "stat", - "title": "Client Success Rate", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 24 - }, - "targets": [ - { - "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-yellow", - "value": 95 - }, - { - "color": "dark-green", - "value": 99 - } - ] - }, - "unit": "percent", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Successful client-facing Switchyard responses in the selected range." - }, - { - "id": 19, - "type": "stat", - "title": "Classifier Fail-Open (Range)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 24 - }, - "targets": [ - { - "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Local classifier failures that safely fell back to the conservative hosted route." - }, - { - "id": 20, - "type": "stat", - "title": "Upstream Errors (Range)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 24 - }, - "targets": [ - { - "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Hosted model attempts that returned errors in the selected range." - }, - { - "id": 21, + "id": 25, "type": "timeseries", "title": "Local Classifier Calls", "datasource": { @@ -1425,7 +1695,7 @@ "description": "Local Qwen routing-classifier activity, split by successful and failed calls." }, { - "id": 22, + "id": 26, "type": "timeseries", "title": "Routing Overhead p95", "datasource": { @@ -1463,7 +1733,7 @@ "description": "95th percentile time Switchyard spends selecting a model before the upstream call." }, { - "id": 23, + "id": 27, "type": "bargauge", "title": "Traffic Lanes (Range)", "datasource": { @@ -1538,7 +1808,7 @@ ] }, { - "id": 24, + "id": 28, "type": "text", "title": "Cost Semantics", "gridPos": { @@ -1554,7 +1824,7 @@ } }, { - "id": 25, + "id": 29, "type": "timeseries", "title": "Hermes Workload CPU (Attribution Proxy)", "datasource": { diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index f24f419c..81ae304a 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -16,7 +16,7 @@ data: { "id": 1, "type": "stat", - "title": "Codex Weekly Remaining", + "title": "Codex 5h Remaining", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -100,12 +100,12 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 2, "type": "stat", - "title": "Codex Spark Weekly Remaining", + "title": "Codex 5h Reset In", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -118,7 +118,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -151,24 +151,12 @@ data: "value": null }, { - "color": "dark-red", + "color": "dark-blue", "value": 0 - }, - { - "color": "dark-orange", - "value": 15 - }, - { - "color": "dark-yellow", - "value": 30 - }, - { - "color": "dark-green", - "value": 50 } ] }, - "unit": "percent", + "unit": "s", "custom": { "displayMode": "auto" }, @@ -189,12 +177,12 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 3, "type": "stat", - "title": "Claude 5h Remaining", + "title": "Codex 7d Remaining", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -207,7 +195,7 @@ data: }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -278,12 +266,12 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 4, "type": "stat", - "title": "Claude 7d Remaining", + "title": "Codex 7d Reset In", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -296,7 +284,210 @@ data: }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 5, + "type": "stat", + "title": "Codex Routed Tokens (24h)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/codex/.+\"}[24h])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 6, + "type": "stat", + "title": "Codex Routed Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/codex/.+\"}[7d])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 7, + "type": "stat", + "title": "Claude 5h Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -367,12 +558,255 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." }, { - "id": 5, + "id": 8, "type": "stat", - "title": "Provider Access Healthy", + "title": "Claude 5h Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "targets": [ + { + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 9, + "type": "stat", + "title": "Claude 7d Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "targets": [ + { + "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Lowest remaining percentage across the provider's active first-party quota buckets for this window. This conservative value shows the limit most likely to constrain new work. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 10, + "type": "stat", + "title": "Claude 7d Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the earliest active first-party quota reset for this provider and window. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss. Not exposed means access is healthy but the credential cannot read this quota; unavailable means access is unhealthy or the snapshot is stale." + }, + { + "id": 11, + "type": "stat", + "title": "Claude Routed Tokens (24h)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -381,7 +815,133 @@ data: "h": 4, "w": 4, "x": 16, - "y": 0 + "y": 4 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/claude/.+\"}[24h])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 12, + "type": "stat", + "title": "Claude Routed Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "targets": [ + { + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_reasoning_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"(route|worker)/claude/.+\"}[7d])) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Prompt, cache-read, cache-creation, reasoning, and completion tokens observed by Switchyard for this provider, including interactive and worker routes." + }, + { + "id": 13, + "type": "stat", + "title": "Provider Access Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 8 }, "targets": [ { @@ -437,7 +997,74 @@ data: "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." }, { - "id": 6, + "id": 14, + "type": "stat", + "title": "Quota Fetch Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 8 + }, + "targets": [ + { + "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 2 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto", + "valueSuffix": " / 2" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Provider quota collectors whose latest refresh completed successfully." + }, + { + "id": 15, "type": "stat", "title": "Oldest Quota Sample", "datasource": { @@ -446,9 +1073,9 @@ data: }, "gridPos": { "h": 4, - "w": 4, - "x": 20, - "y": 0 + "w": 3, + "x": 6, + "y": 8 }, "targets": [ { @@ -507,364 +1134,7 @@ data: "description": "Age of the stalest successful provider quota snapshot." }, { - "id": 7, - "type": "stat", - "title": "Codex Weekly Reset In", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 4 - }, - "targets": [ - { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "s", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 8, - "type": "stat", - "title": "Claude Routed Tokens (24h)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 4 - }, - "targets": [ - { - "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance." - }, - { - "id": 9, - "type": "stat", - "title": "Claude Routed Tokens (7d)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 4 - }, - "targets": [ - { - "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope." - }, - { - "id": 10, - "type": "stat", - "title": "Codex Tokens (Latest Day)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 4 - }, - "targets": [ - { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 11, - "type": "stat", - "title": "Codex Tokens (7d)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 4 - }, - "targets": [ - { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "gray", - "value": null - }, - { - "color": "dark-blue", - "value": 0 - } - ] - }, - "unit": "short", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." - }, - { - "id": 12, + "id": 16, "type": "stat", "title": "Switchyard Requests", "datasource": { @@ -873,9 +1143,9 @@ data: }, "gridPos": { "h": 4, - "w": 4, - "x": 20, - "y": 4 + "w": 3, + "x": 9, + "y": 8 }, "targets": [ { @@ -927,7 +1197,267 @@ data: "description": "Hosted model requests observed by Switchyard in the selected dashboard range." }, { - "id": 13, + "id": 17, + "type": "stat", + "title": "Prompt Cache Share", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." + }, + { + "id": 18, + "type": "stat", + "title": "Client Success Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 8 + }, + "targets": [ + { + "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 99 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful client-facing Switchyard responses in the selected range." + }, + { + "id": 19, + "type": "stat", + "title": "Classifier Fail-Open", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 8 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Local classifier failures that safely fell back to the conservative hosted route." + }, + { + "id": 20, + "type": "stat", + "title": "Upstream Errors", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 8 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model attempts that returned errors in the selected range." + }, + { + "id": 21, "type": "timeseries", "title": "Model Selection Rate", "datasource": { @@ -938,7 +1468,7 @@ data: "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "targets": [ { @@ -965,7 +1495,7 @@ data: "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort." }, { - "id": 14, + "id": 22, "type": "bargauge", "title": "Provider Selections (Range)", "datasource": { @@ -976,7 +1506,7 @@ data: "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "targets": [ { @@ -1040,7 +1570,7 @@ data: ] }, { - "id": 15, + "id": 23, "type": "timeseries", "title": "Token Throughput", "datasource": { @@ -1051,7 +1581,7 @@ data: "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "targets": [ { @@ -1098,7 +1628,7 @@ data: "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls." }, { - "id": 16, + "id": 24, "type": "timeseries", "title": "Model Call p95 Latency", "datasource": { @@ -1109,7 +1639,7 @@ data: "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "targets": [ { @@ -1136,267 +1666,7 @@ data: "description": "95th percentile upstream latency for each selected model route." }, { - "id": 17, - "type": "stat", - "title": "Prompt Cache Share", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 24 - }, - "targets": [ - { - "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(115, 115, 115, 1)", - "value": null - }, - { - "color": "dark-green", - "value": 1 - } - ] - }, - "unit": "percent", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." - }, - { - "id": 18, - "type": "stat", - "title": "Client Success Rate", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 24 - }, - "targets": [ - { - "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-yellow", - "value": 95 - }, - { - "color": "dark-green", - "value": 99 - } - ] - }, - "unit": "percent", - "custom": { - "displayMode": "auto" - }, - "decimals": 1 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Successful client-facing Switchyard responses in the selected range." - }, - { - "id": 19, - "type": "stat", - "title": "Classifier Fail-Open (Range)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 24 - }, - "targets": [ - { - "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Local classifier failures that safely fell back to the conservative hosted route." - }, - { - "id": 20, - "type": "stat", - "title": "Upstream Errors (Range)", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 24 - }, - "targets": [ - { - "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Hosted model attempts that returned errors in the selected range." - }, - { - "id": 21, + "id": 25, "type": "timeseries", "title": "Local Classifier Calls", "datasource": { @@ -1434,7 +1704,7 @@ data: "description": "Local Qwen routing-classifier activity, split by successful and failed calls." }, { - "id": 22, + "id": 26, "type": "timeseries", "title": "Routing Overhead p95", "datasource": { @@ -1472,7 +1742,7 @@ data: "description": "95th percentile time Switchyard spends selecting a model before the upstream call." }, { - "id": 23, + "id": 27, "type": "bargauge", "title": "Traffic Lanes (Range)", "datasource": { @@ -1547,7 +1817,7 @@ data: ] }, { - "id": 24, + "id": 28, "type": "text", "title": "Cost Semantics", "gridPos": { @@ -1563,7 +1833,7 @@ data: } }, { - "id": 25, + "id": 29, "type": "timeseries", "title": "Hermes Workload CPU (Attribution Proxy)", "datasource": { diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index afb7d8e2..cf3c67e0 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -34,12 +34,20 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert dashboard["uid"] == "atlas-ai" assert dashboard["folderUid"] == mod.PRIVATE_FOLDER assert dashboard["refresh"] == "1m" - assert "Codex Weekly Remaining" in panels + assert "Codex 5h Remaining" in panels + assert "Codex 5h Reset In" in panels + assert "Codex 7d Remaining" in panels + assert "Codex 7d Reset In" in panels assert "Claude 5h Remaining" in panels + assert "Claude 5h Reset In" in panels assert "Claude 7d Remaining" in panels + assert "Claude 7d Reset In" in panels + assert "Codex Routed Tokens (24h)" in panels + assert "Codex Routed Tokens (7d)" in panels assert "Claude Routed Tokens (24h)" in panels assert "Claude Routed Tokens (7d)" in panels assert "Provider Access Healthy" in panels + assert "Quota Fetch Healthy" in panels assert "Provider Selections (Range)" in panels assert "Local Classifier Calls" in panels assert "Hermes Workload CPU (Attribution Proxy)" in panels @@ -48,7 +56,9 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert "atlas_ai_provider_authenticated" in expressions assert "switchyard_decisions_total" in expressions assert "switchyard_cached_tokens_total" in expressions - assert 'model=~"route/claude/.+"' in expressions + assert "switchyard_reasoning_tokens_total" in expressions + assert 'model=~"(route|worker)/codex/.+"' in expressions + assert 'model=~"(route|worker)/claude/.+"' in expressions assert "switchyard_model_call_latency_ms_bucket" in expressions assert "/status" not in expressions assert all( @@ -63,13 +73,14 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot(): panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} for title in ( - "Codex Weekly Remaining", - "Codex Spark Weekly Remaining", - "Codex Weekly Reset In", - "Codex Tokens (Latest Day)", - "Codex Tokens (7d)", + "Codex 5h Remaining", + "Codex 5h Reset In", + "Codex 7d Remaining", + "Codex 7d Reset In", "Claude 5h Remaining", + "Claude 5h Reset In", "Claude 7d Remaining", + "Claude 7d Reset In", ): expression = panels[title]["targets"][0]["expr"] assert "atlas_ai_quota_last_success_timestamp_seconds" in expression @@ -77,6 +88,37 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot(): assert "atlas_ai_quota_fetch_success" not in expression +def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric(): + """The first three KPI bands must compare providers without layout gaps.""" + mod = load_module() + dashboard = mod.build_ai_dashboard() + bands = { + y: sorted( + ( + panel + for panel in dashboard["panels"] + if panel["gridPos"]["y"] == y and panel["gridPos"]["h"] == 4 + ), + key=lambda panel: panel["gridPos"]["x"], + ) + for y in (0, 4, 8) + } + + for panels in bands.values(): + assert sum(panel["gridPos"]["w"] for panel in panels) == 24 + assert panels[0]["gridPos"]["x"] == 0 + assert all( + left["gridPos"]["x"] + left["gridPos"]["w"] + == right["gridPos"]["x"] + for left, right in zip(panels, panels[1:]) + ) + + codex_suffixes = [panel["title"].removeprefix("Codex ") for panel in bands[0]] + claude_suffixes = [panel["title"].removeprefix("Claude ") for panel in bands[4]] + assert codex_suffixes == claude_suffixes + assert len(bands[8]) == 8 + + def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution(): mod = load_module() dashboard = mod.build_ai_dashboard()