titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): compare provider quota windows

Browse Source
This commit is contained in:
jenkins 2026-08-23 09:48:11 -03:00
parent ced9a24cd9
commit a019ecd556
4 changed files with 2134 additions and 1458 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

328
scripts/render/dashboards_render_atlas.py
View File

@ -5198,6 +5198,7 @@ def build_ai_dashboard():
*,
unit="percent",
authenticated_expr=None,
description=None,
):
thresholds = remaining_thresholds
if unit != "percent":
@ -5210,6 +5211,12 @@ def build_ai_dashboard():
}
if authenticated_expr:
expr = f"({expr}) or on() (({authenticated_expr}) * 0 - 2)"
quota_description = (
"A last-good snapshot remains visible for up to 20 minutes through "
"a transient fetch miss. Not exposed means access is healthy but the "
"credential cannot read this quota; unavailable means access is "
"unhealthy or the snapshot is stale."
)
panel = stat_panel(
panel_id,
title,
@ -5219,7 +5226,11 @@ def build_ai_dashboard():
decimals=1,
thresholds=thresholds,
instant=True,
description="Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale.",
description=(
f"{description} {quota_description}"
if description
else f"Live first-party CLI account telemetry. {quota_description}"
),
)
panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping
return panel
@ -5235,59 +5246,166 @@ def build_ai_dashboard():
openai_snapshot = snapshot_fresh("openai")
anthropic_snapshot = snapshot_fresh("anthropic")
openai_authenticated = 'atlas_ai_provider_authenticated{provider="openai"} == 1'
anthropic_authenticated = (
'atlas_ai_provider_authenticated{provider="anthropic"} == 1'
)
quota = "atlas_ai_quota_remaining_percent"
reset = "atlas_ai_quota_reset_timestamp_seconds"
claude_models = 'model=~"route/claude/.+"'
def claude_routed_tokens(window):
"""Count the disjoint Claude token classes Switchyard observed."""
def provider_quota(metric, provider, window):
"""Aggregate provider quota buckets into one conservative KPI."""
return (
f'min by (provider) ({metric}{{provider="{provider}",'
f'window="{window}"}})'
)
def provider_routed_tokens(provider, window):
"""Count equivalent routed token classes for one hosted provider."""
provider_models = f'model=~"(route|worker)/{provider}/.+"'
metrics = (
"switchyard_prompt_tokens_total",
"switchyard_cached_tokens_total",
"switchyard_cache_creation_tokens_total",
"switchyard_reasoning_tokens_total",
"switchyard_completion_tokens_total",
)
terms = [
f'(sum(increase({metric}{{{claude_models}}}[{window}])) or on() vector(0))'
f'(sum(increase({metric}{{{provider_models}}}[{window}])) '
"or on() vector(0))"
for metric in metrics
]
return " + ".join(terms)
remaining_description = (
"Lowest remaining percentage across the provider's active first-party "
"quota buckets for this window. This conservative value shows the limit "
"most likely to constrain new work."
)
reset_description = (
"Time until the earliest active first-party quota reset for this provider "
"and window."
)
routed_tokens_description = (
"Prompt, cache-read, cache-creation, reasoning, and completion tokens "
"observed by Switchyard for this provider, including interactive and "
"worker routes."
)
panels = [
quota_stat(
1,
"Codex Weekly Remaining",
f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_snapshot})',
"Codex 5h Remaining",
f'{provider_quota(quota, "openai", "five_hour")} and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 0, "y": 0},
authenticated_expr=openai_authenticated,
description=remaining_description,
),
quota_stat(
2,
"Codex Spark Weekly Remaining",
f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_snapshot})',
"Codex 5h Reset In",
f'clamp_min({provider_quota(reset, "openai", "five_hour")} - time(), 0) and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 4, "y": 0},
unit="s",
authenticated_expr=openai_authenticated,
description=reset_description,
),
quota_stat(
3,
"Claude 5h Remaining",
f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_snapshot})',
"Codex 7d Remaining",
f'{provider_quota(quota, "openai", "seven_day")} and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 8, "y": 0},
authenticated_expr=anthropic_authenticated,
authenticated_expr=openai_authenticated,
description=remaining_description,
),
quota_stat(
4,
"Claude 7d Remaining",
f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_snapshot})',
"Codex 7d Reset In",
f'clamp_min({provider_quota(reset, "openai", "seven_day")} - time(), 0) and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 12, "y": 0},
authenticated_expr=anthropic_authenticated,
unit="s",
authenticated_expr=openai_authenticated,
description=reset_description,
),
stat_panel(
5,
"Codex Routed Tokens (24h)",
provider_routed_tokens("codex", "24h"),
{"h": 4, "w": 4, "x": 16, "y": 0},
unit="short",
decimals=0,
instant=True,
description=routed_tokens_description,
),
stat_panel(
6,
"Codex Routed Tokens (7d)",
provider_routed_tokens("codex", "7d"),
{"h": 4, "w": 4, "x": 20, "y": 0},
unit="short",
decimals=0,
instant=True,
description=routed_tokens_description,
),
quota_stat(
7,
"Claude 5h Remaining",
f'{provider_quota(quota, "anthropic", "five_hour")} and on(provider) ({anthropic_snapshot})',
{"h": 4, "w": 4, "x": 0, "y": 4},
authenticated_expr=anthropic_authenticated,
description=remaining_description,
),
quota_stat(
8,
"Claude 5h Reset In",
f'clamp_min({provider_quota(reset, "anthropic", "five_hour")} - time(), 0) and on(provider) ({anthropic_snapshot})',
{"h": 4, "w": 4, "x": 4, "y": 4},
unit="s",
authenticated_expr=anthropic_authenticated,
description=reset_description,
),
quota_stat(
9,
"Claude 7d Remaining",
f'{provider_quota(quota, "anthropic", "seven_day")} and on(provider) ({anthropic_snapshot})',
{"h": 4, "w": 4, "x": 8, "y": 4},
authenticated_expr=anthropic_authenticated,
description=remaining_description,
),
quota_stat(
10,
"Claude 7d Reset In",
f'clamp_min({provider_quota(reset, "anthropic", "seven_day")} - time(), 0) and on(provider) ({anthropic_snapshot})',
{"h": 4, "w": 4, "x": 12, "y": 4},
unit="s",
authenticated_expr=anthropic_authenticated,
description=reset_description,
),
stat_panel(
11,
"Claude Routed Tokens (24h)",
provider_routed_tokens("claude", "24h"),
{"h": 4, "w": 4, "x": 16, "y": 4},
unit="short",
decimals=0,
instant=True,
description=routed_tokens_description,
),
stat_panel(
12,
"Claude Routed Tokens (7d)",
provider_routed_tokens("claude", "7d"),
{"h": 4, "w": 4, "x": 20, "y": 4},
unit="short",
decimals=0,
instant=True,
description=routed_tokens_description,
),
stat_panel(
13,
"Provider Access Healthy",
"sum(atlas_ai_provider_authenticated) or on() vector(0)",
{"h": 4, "w": 4, "x": 16, "y": 0},
{"h": 4, "w": 3, "x": 0, "y": 8},
instant=True,
thresholds={
"mode": "absolute",
@ -5301,10 +5419,27 @@ def build_ai_dashboard():
description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.",
),
stat_panel(
6,
14,
"Quota Fetch Healthy",
"sum(atlas_ai_quota_fetch_success) or on() vector(0)",
{"h": 4, "w": 3, "x": 3, "y": 8},
instant=True,
thresholds={
"mode": "absolute",
"steps": [
{"color": "red", "value": None},
{"color": "yellow", "value": 1},
{"color": "green", "value": 2},
],
},
value_suffix=" / 2",
description="Provider quota collectors whose latest refresh completed successfully.",
),
stat_panel(
15,
"Oldest Quota Sample",
"max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)",
{"h": 4, "w": 4, "x": 20, "y": 0},
{"h": 4, "w": 3, "x": 6, "y": 8},
unit="s",
instant=True,
thresholds={
@ -5318,108 +5453,21 @@ def build_ai_dashboard():
},
description="Age of the stalest successful provider quota snapshot.",
),
quota_stat(
7,
"Codex Weekly Reset In",
f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 0, "y": 4},
unit="s",
),
stat_panel(
8,
"Claude Routed Tokens (24h)",
claude_routed_tokens("24h"),
{"h": 4, "w": 4, "x": 4, "y": 4},
unit="short",
decimals=0,
instant=True,
description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance.",
),
stat_panel(
9,
"Claude Routed Tokens (7d)",
claude_routed_tokens("7d"),
{"h": 4, "w": 4, "x": 8, "y": 4},
unit="short",
decimals=0,
instant=True,
description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope.",
),
quota_stat(
10,
"Codex Tokens (Latest Day)",
f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 12, "y": 4},
unit="short",
),
quota_stat(
11,
"Codex Tokens (7d)",
f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_snapshot})',
{"h": 4, "w": 4, "x": 16, "y": 4},
unit="short",
),
stat_panel(
12,
16,
"Switchyard Requests",
"sum(increase(switchyard_requests_total[$__range])) or on() vector(0)",
{"h": 4, "w": 4, "x": 20, "y": 4},
{"h": 4, "w": 3, "x": 9, "y": 8},
unit="short",
decimals=0,
instant=True,
description="Hosted model requests observed by Switchyard in the selected dashboard range.",
),
timeseries_panel(
13,
"Model Selection Rate",
"sum by (selected_model) (rate(switchyard_decisions_total[5m]))",
{"h": 8, "w": 12, "x": 0, "y": 8},
unit="reqps",
legend="{{selected_model}}",
legend_placement="right",
description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.",
),
bargauge_panel(
14,
"Provider Selections (Range)",
'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))',
{"h": 8, "w": 12, "x": 12, "y": 8},
unit="short",
legend="{{provider}}",
instant=True,
include_color=False,
description="Switchyard selections grouped by provider over the selected dashboard range.",
),
timeseries_panel(
15,
"Token Throughput",
None,
{"h": 8, "w": 12, "x": 0, "y": 16},
unit="tps",
targets=[
{"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"},
{"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"},
{"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"},
{"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"},
{"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"},
],
description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.",
),
timeseries_panel(
16,
"Model Call p95 Latency",
"histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))",
{"h": 8, "w": 12, "x": 12, "y": 16},
unit="ms",
legend="{{model}}",
legend_placement="right",
description="95th percentile upstream latency for each selected model route.",
),
stat_panel(
17,
"Prompt Cache Share",
"100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)",
{"h": 4, "w": 6, "x": 0, "y": 24},
{"h": 4, "w": 3, "x": 12, "y": 8},
unit="percent",
decimals=1,
description="Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.",
@ -5428,7 +5476,7 @@ def build_ai_dashboard():
18,
"Client Success Rate",
'100 * sum(increase(switchyard_client_responses_total{outcome="success"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)',
{"h": 4, "w": 6, "x": 6, "y": 24},
{"h": 4, "w": 3, "x": 15, "y": 8},
unit="percent",
decimals=1,
thresholds={
@ -5443,9 +5491,9 @@ def build_ai_dashboard():
),
stat_panel(
19,
"Classifier Fail-Open (Range)",
"Classifier Fail-Open",
"sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)",
{"h": 4, "w": 6, "x": 12, "y": 24},
{"h": 4, "w": 3, "x": 18, "y": 8},
decimals=0,
thresholds={
"mode": "absolute",
@ -5459,9 +5507,9 @@ def build_ai_dashboard():
),
stat_panel(
20,
"Upstream Errors (Range)",
"Upstream Errors",
"sum(increase(switchyard_errors_total[$__range])) or on() vector(0)",
{"h": 4, "w": 6, "x": 18, "y": 24},
{"h": 4, "w": 3, "x": 21, "y": 8},
decimals=0,
thresholds={
"mode": "absolute",
@ -5475,6 +5523,52 @@ def build_ai_dashboard():
),
timeseries_panel(
21,
"Model Selection Rate",
"sum by (selected_model) (rate(switchyard_decisions_total[5m]))",
{"h": 8, "w": 12, "x": 0, "y": 12},
unit="reqps",
legend="{{selected_model}}",
legend_placement="right",
description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.",
),
bargauge_panel(
22,
"Provider Selections (Range)",
'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))',
{"h": 8, "w": 12, "x": 12, "y": 12},
unit="short",
legend="{{provider}}",
instant=True,
include_color=False,
description="Switchyard selections grouped by provider over the selected dashboard range.",
),
timeseries_panel(
23,
"Token Throughput",
None,
{"h": 8, "w": 12, "x": 0, "y": 20},
unit="tps",
targets=[
{"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"},
{"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"},
{"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"},
{"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"},
{"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"},
],
description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.",
),
timeseries_panel(
24,
"Model Call p95 Latency",
"histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))",
{"h": 8, "w": 12, "x": 12, "y": 20},
unit="ms",
legend="{{model}}",
legend_placement="right",
description="95th percentile upstream latency for each selected model route.",
),
timeseries_panel(
25,
"Local Classifier Calls",
'sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~"qwen.*"}[5m]))',
{"h": 8, "w": 8, "x": 0, "y": 28},
@ -5483,7 +5577,7 @@ def build_ai_dashboard():
description="Local Qwen routing-classifier activity, split by successful and failed calls.",
),
timeseries_panel(
22,
26,
"Routing Overhead p95",
"histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))",
{"h": 8, "w": 8, "x": 8, "y": 28},
@ -5492,7 +5586,7 @@ def build_ai_dashboard():
description="95th percentile time Switchyard spends selecting a model before the upstream call.",
),
bargauge_panel(
23,
27,
"Traffic Lanes (Range)",
'sum by (lane) (label_replace(increase(switchyard_requests_total{model=~"(route|worker)/.*"}[$__range]), "lane", "$1", "model", "^(route|worker)/.*"))',
{"h": 8, "w": 8, "x": 16, "y": 28},
@ -5503,13 +5597,13 @@ def build_ai_dashboard():
description="Request volume split between interactive route traffic and durable worker traffic.",
),
text_panel(
24,
28,
"Cost Semantics",
"Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage.",
{"h": 8, "w": 12, "x": 0, "y": 36},
),
timeseries_panel(
25,
29,
"Hermes Workload CPU (Attribution Proxy)",
'sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace="hermes",pod=~"hermes-(agent|chat-tenant|switchyard|model-gate).*",container!="",image!=""}[5m]))',
{"h": 8, "w": 12, "x": 12, "y": 36},

1604
services/monitoring/dashboards/atlas-ai.json
View File

File diff suppressed because it is too large Load Diff

1604
services/monitoring/grafana-dashboard-ai.yaml
View File

File diff suppressed because it is too large Load Diff

56
testing/tests/test_atlas_ai_dashboard.py
View File

@ -34,12 +34,20 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
assert dashboard["uid"] == "atlas-ai"
assert dashboard["folderUid"] == mod.PRIVATE_FOLDER
assert dashboard["refresh"] == "1m"
assert "Codex Weekly Remaining" in panels
assert "Codex 5h Remaining" in panels
assert "Codex 5h Reset In" in panels
assert "Codex 7d Remaining" in panels
assert "Codex 7d Reset In" in panels
assert "Claude 5h Remaining" in panels
assert "Claude 5h Reset In" in panels
assert "Claude 7d Remaining" in panels
assert "Claude 7d Reset In" in panels
assert "Codex Routed Tokens (24h)" in panels
assert "Codex Routed Tokens (7d)" in panels
assert "Claude Routed Tokens (24h)" in panels
assert "Claude Routed Tokens (7d)" in panels
assert "Provider Access Healthy" in panels
assert "Quota Fetch Healthy" in panels
assert "Provider Selections (Range)" in panels
assert "Local Classifier Calls" in panels
assert "Hermes Workload CPU (Attribution Proxy)" in panels
@ -48,7 +56,9 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
assert "atlas_ai_provider_authenticated" in expressions
assert "switchyard_decisions_total" in expressions
assert "switchyard_cached_tokens_total" in expressions
assert 'model=~"route/claude/.+"' in expressions
assert "switchyard_reasoning_tokens_total" in expressions
assert 'model=~"(route|worker)/codex/.+"' in expressions
assert 'model=~"(route|worker)/claude/.+"' in expressions
assert "switchyard_model_call_latency_ms_bucket" in expressions
assert "/status" not in expressions
assert all(
@ -63,13 +73,14 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
for title in (
"Codex Weekly Remaining",
"Codex Spark Weekly Remaining",
"Codex Weekly Reset In",
"Codex Tokens (Latest Day)",
"Codex Tokens (7d)",
"Codex 5h Remaining",
"Codex 5h Reset In",
"Codex 7d Remaining",
"Codex 7d Reset In",
"Claude 5h Remaining",
"Claude 5h Reset In",
"Claude 7d Remaining",
"Claude 7d Reset In",
):
expression = panels[title]["targets"][0]["expr"]
assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
@ -77,6 +88,37 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
assert "atlas_ai_quota_fetch_success" not in expression
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
"""The first three KPI bands must compare providers without layout gaps."""
mod = load_module()
dashboard = mod.build_ai_dashboard()
bands = {
y: sorted(
(
panel
for panel in dashboard["panels"]
if panel["gridPos"]["y"] == y and panel["gridPos"]["h"] == 4
),
key=lambda panel: panel["gridPos"]["x"],
)
for y in (0, 4, 8)
}
for panels in bands.values():
assert sum(panel["gridPos"]["w"] for panel in panels) == 24
assert panels[0]["gridPos"]["x"] == 0
assert all(
left["gridPos"]["x"] + left["gridPos"]["w"]
== right["gridPos"]["x"]
for left, right in zip(panels, panels[1:])
)
codex_suffixes = [panel["title"].removeprefix("Codex ") for panel in bands[0]]
claude_suffixes = [panel["title"].removeprefix("Claude ") for panel in bands[4]]
assert codex_suffixes == claude_suffixes
assert len(bands[8]) == 8
def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution():
mod = load_module()
dashboard = mod.build_ai_dashboard()