titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): measure Claude routed usage

Browse Source
This commit is contained in:
jenkins 2026-08-22 23:38:40 -03:00
parent 69ad76bebc
commit a7e09f9719
4 changed files with 69 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
scripts/render/dashboards_render_atlas.py
View File

@ -5231,6 +5231,22 @@ def build_ai_dashboard():
)
quota = "atlas_ai_quota_remaining_percent"
reset = "atlas_ai_quota_reset_timestamp_seconds"
claude_models = 'model=~"route/claude/.+"'
def claude_routed_tokens(window):
"""Count the disjoint Claude token classes Switchyard observed."""
metrics = (
"switchyard_prompt_tokens_total",
"switchyard_cached_tokens_total",
"switchyard_cache_creation_tokens_total",
"switchyard_completion_tokens_total",
)
terms = [
f'(sum(increase({metric}{{{claude_models}}}[{window}])) or on() vector(0))'
for metric in metrics
]
return " + ".join(terms)
panels = [
quota_stat(
1,
@ -5300,21 +5316,25 @@ def build_ai_dashboard():
{"h": 4, "w": 4, "x": 0, "y": 4},
unit="s",
),
quota_stat(
stat_panel(
8,
"Claude 5h Reset In",
f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})',
"Claude Routed Tokens (24h)",
claude_routed_tokens("24h"),
{"h": 4, "w": 4, "x": 4, "y": 4},
unit="s",
authenticated_expr=anthropic_authenticated,
unit="short",
decimals=0,
instant=True,
description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance.",
),
quota_stat(
stat_panel(
9,
"Claude 7d Reset In",
f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})',
"Claude Routed Tokens (7d)",
claude_routed_tokens("7d"),
{"h": 4, "w": 4, "x": 8, "y": 4},
unit="s",
authenticated_expr=anthropic_authenticated,
unit="short",
decimals=0,
instant=True,
description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope.",
),
quota_stat(
10,

64
services/monitoring/dashboards/atlas-ai.json
View File

@ -577,7 +577,7 @@
{
"id": 8,
"type": "stat",
"title": "Claude 5h Reset In",
"title": "Claude Routed Tokens (24h)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -590,7 +590,7 @@
},
"targets": [
{
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))",
"refId": "A",
"instant": true
}
@ -600,39 +600,25 @@
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-2": {
"text": "not exposed",
"color": "dark-blue"
},
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "gray",
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "dark-blue",
"value": 0
"color": "dark-green",
"value": 1
}
]
},
"unit": "s",
"unit": "short",
"custom": {
"displayMode": "auto"
},
"decimals": 1
"decimals": 0
},
"overrides": []
},
@ -649,12 +635,12 @@
},
"textMode": "value"
},
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
"description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance."
},
{
"id": 9,
"type": "stat",
"title": "Claude 7d Reset In",
"title": "Claude Routed Tokens (7d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -667,7 +653,7 @@
},
"targets": [
{
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))",
"refId": "A",
"instant": true
}
@ -677,39 +663,25 @@
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-2": {
"text": "not exposed",
"color": "dark-blue"
},
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "gray",
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "dark-blue",
"value": 0
"color": "dark-green",
"value": 1
}
]
},
"unit": "s",
"unit": "short",
"custom": {
"displayMode": "auto"
},
"decimals": 1
"decimals": 0
},
"overrides": []
},
@ -726,7 +698,7 @@
},
"textMode": "value"
},
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
"description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope."
},
{
"id": 10,

64
services/monitoring/grafana-dashboard-ai.yaml
View File

@ -586,7 +586,7 @@ data:
{
"id": 8,
"type": "stat",
"title": "Claude 5h Reset In",
"title": "Claude Routed Tokens (24h)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -599,7 +599,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))",
"refId": "A",
"instant": true
}
@ -609,39 +609,25 @@ data:
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-2": {
"text": "not exposed",
"color": "dark-blue"
},
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "gray",
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "dark-blue",
"value": 0
"color": "dark-green",
"value": 1
}
]
},
"unit": "s",
"unit": "short",
"custom": {
"displayMode": "auto"
},
"decimals": 1
"decimals": 0
},
"overrides": []
},
@ -658,12 +644,12 @@ data:
},
"textMode": "value"
},
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
"description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance."
},
{
"id": 9,
"type": "stat",
"title": "Claude 7d Reset In",
"title": "Claude Routed Tokens (7d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -676,7 +662,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))",
"refId": "A",
"instant": true
}
@ -686,39 +672,25 @@ data:
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-2": {
"text": "not exposed",
"color": "dark-blue"
},
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "gray",
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "dark-blue",
"value": 0
"color": "dark-green",
"value": 1
}
]
},
"unit": "s",
"unit": "short",
"custom": {
"displayMode": "auto"
},
"decimals": 1
"decimals": 0
},
"overrides": []
},
@ -735,7 +707,7 @@ data:
},
"textMode": "value"
},
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
"description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope."
},
{
"id": 10,

3
testing/tests/test_atlas_ai_dashboard.py
View File

@ -37,6 +37,8 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
assert "Codex Weekly Remaining" in panels
assert "Claude 5h Remaining" in panels
assert "Claude 7d Remaining" in panels
assert "Claude Routed Tokens (24h)" in panels
assert "Claude Routed Tokens (7d)" in panels
assert "Provider Access Healthy" in panels
assert "Provider Selections (Range)" in panels
assert "Local Classifier Calls" in panels
@ -46,6 +48,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
assert "atlas_ai_provider_authenticated" in expressions
assert "switchyard_decisions_total" in expressions
assert "switchyard_cached_tokens_total" in expressions
assert 'model=~"route/claude/.+"' in expressions
assert "switchyard_model_call_latency_ms_bucket" in expressions
assert "/status" not in expressions
assert all(