From a7e09f9719429731b410a124e0884f3d85ebc62e Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 22 Aug 2026 23:38:40 -0300 Subject: [PATCH] monitoring(ai): measure Claude routed usage --- scripts/render/dashboards_render_atlas.py | 40 +++++++++--- services/monitoring/dashboards/atlas-ai.json | 64 ++++++------------- services/monitoring/grafana-dashboard-ai.yaml | 64 ++++++------------- testing/tests/test_atlas_ai_dashboard.py | 3 + 4 files changed, 69 insertions(+), 102 deletions(-) diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index e0986bdc..adac0d25 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5231,6 +5231,22 @@ def build_ai_dashboard(): ) quota = "atlas_ai_quota_remaining_percent" reset = "atlas_ai_quota_reset_timestamp_seconds" + claude_models = 'model=~"route/claude/.+"' + + def claude_routed_tokens(window): + """Count the disjoint Claude token classes Switchyard observed.""" + metrics = ( + "switchyard_prompt_tokens_total", + "switchyard_cached_tokens_total", + "switchyard_cache_creation_tokens_total", + "switchyard_completion_tokens_total", + ) + terms = [ + f'(sum(increase({metric}{{{claude_models}}}[{window}])) or on() vector(0))' + for metric in metrics + ] + return " + ".join(terms) + panels = [ quota_stat( 1, @@ -5300,21 +5316,25 @@ def build_ai_dashboard(): {"h": 4, "w": 4, "x": 0, "y": 4}, unit="s", ), - quota_stat( + stat_panel( 8, - "Claude 5h Reset In", - f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})', + "Claude Routed Tokens (24h)", + claude_routed_tokens("24h"), {"h": 4, "w": 4, "x": 4, "y": 4}, - unit="s", - authenticated_expr=anthropic_authenticated, + unit="short", + decimals=0, + instant=True, + description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance.", ), - quota_stat( + stat_panel( 9, - "Claude 7d Reset In", - f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})', + "Claude Routed Tokens (7d)", + claude_routed_tokens("7d"), {"h": 4, "w": 4, "x": 8, "y": 4}, - unit="s", - authenticated_expr=anthropic_authenticated, + unit="short", + decimals=0, + instant=True, + description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope.", ), quota_stat( 10, diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index bc9e1d12..f745d809 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -577,7 +577,7 @@ { "id": 8, "type": "stat", - "title": "Claude 5h Reset In", + "title": "Claude Routed Tokens (24h)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -590,7 +590,7 @@ }, "targets": [ { - "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))", "refId": "A", "instant": true } @@ -600,39 +600,25 @@ "color": { "mode": "thresholds" }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "gray", + "color": "rgba(115, 115, 115, 1)", "value": null }, { - "color": "dark-blue", - "value": 0 + "color": "dark-green", + "value": 1 } ] }, - "unit": "s", + "unit": "short", "custom": { "displayMode": "auto" }, - "decimals": 1 + "decimals": 0 }, "overrides": [] }, @@ -649,12 +635,12 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance." }, { "id": 9, "type": "stat", - "title": "Claude 7d Reset In", + "title": "Claude Routed Tokens (7d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -667,7 +653,7 @@ }, "targets": [ { - "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))", "refId": "A", "instant": true } @@ -677,39 +663,25 @@ "color": { "mode": "thresholds" }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "gray", + "color": "rgba(115, 115, 115, 1)", "value": null }, { - "color": "dark-blue", - "value": 0 + "color": "dark-green", + "value": 1 } ] }, - "unit": "s", + "unit": "short", "custom": { "displayMode": "auto" }, - "decimals": 1 + "decimals": 0 }, "overrides": [] }, @@ -726,7 +698,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope." }, { "id": 10, diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index f078bcf0..ad209720 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -586,7 +586,7 @@ data: { "id": 8, "type": "stat", - "title": "Claude 5h Reset In", + "title": "Claude Routed Tokens (24h)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -599,7 +599,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))", "refId": "A", "instant": true } @@ -609,39 +609,25 @@ data: "color": { "mode": "thresholds" }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "gray", + "color": "rgba(115, 115, 115, 1)", "value": null }, { - "color": "dark-blue", - "value": 0 + "color": "dark-green", + "value": 1 } ] }, - "unit": "s", + "unit": "short", "custom": { "displayMode": "auto" }, - "decimals": 1 + "decimals": 0 }, "overrides": [] }, @@ -658,12 +644,12 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance." }, { "id": 9, "type": "stat", - "title": "Claude 7d Reset In", + "title": "Claude Routed Tokens (7d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -676,7 +662,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))", "refId": "A", "instant": true } @@ -686,39 +672,25 @@ data: "color": { "mode": "thresholds" }, - "mappings": [ - { - "type": "value", - "options": { - "-2": { - "text": "not exposed", - "color": "dark-blue" - }, - "-1": { - "text": "unavailable", - "color": "gray" - } - } - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "gray", + "color": "rgba(115, 115, 115, 1)", "value": null }, { - "color": "dark-blue", - "value": 0 + "color": "dark-green", + "value": 1 } ] }, - "unit": "s", + "unit": "short", "custom": { "displayMode": "auto" }, - "decimals": 1 + "decimals": 0 }, "overrides": [] }, @@ -735,7 +707,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope." }, { "id": 10, diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index 5d9e205e..455b6ead 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -37,6 +37,8 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert "Codex Weekly Remaining" in panels assert "Claude 5h Remaining" in panels assert "Claude 7d Remaining" in panels + assert "Claude Routed Tokens (24h)" in panels + assert "Claude Routed Tokens (7d)" in panels assert "Provider Access Healthy" in panels assert "Provider Selections (Range)" in panels assert "Local Classifier Calls" in panels @@ -46,6 +48,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert "atlas_ai_provider_authenticated" in expressions assert "switchyard_decisions_total" in expressions assert "switchyard_cached_tokens_total" in expressions + assert 'model=~"route/claude/.+"' in expressions assert "switchyard_model_call_latency_ms_bucket" in expressions assert "/status" not in expressions assert all(