titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): preserve quota data across rollouts

Browse Source
This commit is contained in:
jenkins 2026-08-25 18:33:18 -03:00
parent 6f783b7778
commit 342677dde7
4 changed files with 150 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scripts/render/dashboards_render_atlas.py
View File

@ -5170,6 +5170,7 @@ def build_testing_dashboard():
def build_ai_dashboard(): def build_ai_dashboard():
"""Build the internal AI quota, routing, and workload dashboard.""" """Build the internal AI quota, routing, and workload dashboard."""
quota_snapshot_range = "20m"
remaining_thresholds = { remaining_thresholds = {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -5253,18 +5254,25 @@ def build_ai_dashboard():
def snapshot_fresh(provider): def snapshot_fresh(provider):
"""Require authenticated access and a successful snapshot under 20m old.""" """Require authenticated access and a successful snapshot under 20m old."""
authenticated = f'atlas_ai_provider_authenticated{{provider="{provider}"}} == 1' authenticated = (
"last_over_time(atlas_ai_provider_authenticated"
f'{{provider="{provider}"}}[{quota_snapshot_range}]) == 1'
)
last_success = ( last_success = (
"time() - atlas_ai_quota_last_success_timestamp_seconds" "time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds"
f'{{provider="{provider}"}} < 1200' f'{{provider="{provider}"}}[{quota_snapshot_range}]) < 1200'
) )
return f"({authenticated}) and on(provider) ({last_success})" return f"({authenticated}) and on(provider) ({last_success})"
openai_snapshot = snapshot_fresh("openai") openai_snapshot = snapshot_fresh("openai")
anthropic_snapshot = snapshot_fresh("anthropic") anthropic_snapshot = snapshot_fresh("anthropic")
openai_authenticated = 'atlas_ai_provider_authenticated{provider="openai"} == 1' openai_authenticated = (
"last_over_time(atlas_ai_provider_authenticated"
f'{{provider="openai"}}[{quota_snapshot_range}]) == 1'
)
anthropic_authenticated = ( anthropic_authenticated = (
'atlas_ai_provider_authenticated{provider="anthropic"} == 1' "last_over_time(atlas_ai_provider_authenticated"
f'{{provider="anthropic"}}[{quota_snapshot_range}]) == 1'
) )
quota = "atlas_ai_quota_remaining_percent" quota = "atlas_ai_quota_remaining_percent"
reset = "atlas_ai_quota_reset_timestamp_seconds" reset = "atlas_ai_quota_reset_timestamp_seconds"
@ -5272,8 +5280,9 @@ def build_ai_dashboard():
def provider_quota(metric, provider, window): def provider_quota(metric, provider, window):
"""Aggregate provider quota buckets into one conservative KPI.""" """Aggregate provider quota buckets into one conservative KPI."""
return ( return (
f'min by (provider) ({metric}{{provider="{provider}",' "min by (provider) (last_over_time("
f'window="{window}"}})' f'{metric}{{provider="{provider}",window="{window}"}}'
f"[{quota_snapshot_range}]))"
) )
def provider_routed_tokens(provider, window): def provider_routed_tokens(provider, window):
@ -5436,7 +5445,7 @@ def build_ai_dashboard():
stat_panel( stat_panel(
13, 13,
"Provider Access Healthy", "Provider Access Healthy",
"sum(atlas_ai_provider_authenticated) or on() vector(0)", f"sum(last_over_time(atlas_ai_provider_authenticated[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 0, "y": 8}, {"h": 4, "w": 3, "x": 0, "y": 8},
instant=True, instant=True,
thresholds={ thresholds={
@ -5448,12 +5457,12 @@ def build_ai_dashboard():
], ],
}, },
value_suffix=" / 2", value_suffix=" / 2",
description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.", description="Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access.",
), ),
stat_panel( stat_panel(
14, 14,
"Quota Fetch Healthy", "Quota Fetch Healthy",
"sum(atlas_ai_quota_fetch_success) or on() vector(0)", f"sum(last_over_time(atlas_ai_quota_fetch_success[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 3, "y": 8}, {"h": 4, "w": 3, "x": 3, "y": 8},
instant=True, instant=True,
thresholds={ thresholds={
@ -5465,12 +5474,12 @@ def build_ai_dashboard():
], ],
}, },
value_suffix=" / 2", value_suffix=" / 2",
description="Provider quota collectors whose latest refresh completed successfully.", description="Provider quota collectors whose latest refresh observed within 20 minutes completed successfully.",
), ),
stat_panel( stat_panel(
15, 15,
"Oldest Quota Sample", "Oldest Quota Sample",
"max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", f"max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 6, "y": 8}, {"h": 4, "w": 3, "x": 6, "y": 8},
unit="s", unit="s",
instant=True, instant=True,
@ -5645,6 +5654,20 @@ def build_ai_dashboard():
description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.",
), ),
] ]
for panel in panels:
if panel["title"] in {
"Provider Access Healthy",
"Quota Fetch Healthy",
"Oldest Quota Sample",
}:
panel["fieldConfig"]["defaults"]["mappings"] = [
{
"type": "value",
"options": {
"-1": {"text": "unavailable", "color": "gray"},
},
}
]
return { return {
"uid": "atlas-ai", "uid": "atlas-ai",
"title": "Atlas AI Operations", "title": "Atlas AI Operations",

64
services/monitoring/dashboards/atlas-ai.json
View File

@ -20,7 +20,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -109,7 +109,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -186,7 +186,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -275,7 +275,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -478,7 +478,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -567,7 +567,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -644,13 +644,13 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"legendFormat": "Overall weekly", "legendFormat": "Overall weekly",
"instant": true "instant": true
}, },
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "B", "refId": "B",
"legendFormat": "Fable 5 weekly", "legendFormat": "Fable 5 weekly",
"instant": true "instant": true
@ -740,7 +740,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -943,7 +943,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", "expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -953,7 +953,17 @@
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -992,7 +1002,7 @@
}, },
"textMode": "value" "textMode": "value"
}, },
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." "description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access."
}, },
{ {
"id": 14, "id": 14,
@ -1010,7 +1020,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", "expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -1020,7 +1030,17 @@
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -1059,7 +1079,7 @@
}, },
"textMode": "value" "textMode": "value"
}, },
"description": "Provider quota collectors whose latest refresh completed successfully." "description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully."
}, },
{ {
"id": 15, "id": 15,
@ -1077,7 +1097,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", "expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -1087,7 +1107,17 @@
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [

64
services/monitoring/grafana-dashboard-ai.yaml
View File

@ -29,7 +29,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -118,7 +118,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -195,7 +195,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -284,7 +284,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -487,7 +487,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -576,7 +576,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -653,13 +653,13 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"legendFormat": "Overall weekly", "legendFormat": "Overall weekly",
"instant": true "instant": true
}, },
{ {
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "B", "refId": "B",
"legendFormat": "Fable 5 weekly", "legendFormat": "Fable 5 weekly",
"instant": true "instant": true
@ -749,7 +749,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -952,7 +952,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", "expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -962,7 +962,17 @@ data:
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -1001,7 +1011,7 @@ data:
}, },
"textMode": "value" "textMode": "value"
}, },
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." "description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access."
}, },
{ {
"id": 14, "id": 14,
@ -1019,7 +1029,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", "expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -1029,7 +1039,17 @@ data:
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -1068,7 +1088,7 @@ data:
}, },
"textMode": "value" "textMode": "value"
}, },
"description": "Provider quota collectors whose latest refresh completed successfully." "description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully."
}, },
{ {
"id": 15, "id": 15,
@ -1086,7 +1106,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", "expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)",
"refId": "A", "refId": "A",
"instant": true "instant": true
} }
@ -1096,7 +1116,17 @@ data:
"color": { "color": {
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [

21
testing/tests/test_atlas_ai_dashboard.py
View File

@ -103,10 +103,31 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
): ):
expression = panels[title]["targets"][0]["expr"] expression = panels[title]["targets"][0]["expr"]
assert "atlas_ai_quota_last_success_timestamp_seconds" in expression assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
assert "last_over_time" in expression
assert "[20m]" in expression
assert "1200" in expression assert "1200" in expression
assert "atlas_ai_quota_fetch_success" not in expression assert "atlas_ai_quota_fetch_success" not in expression
def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes():
"""Recent samples bridge rollouts, while missing telemetry stays explicit."""
mod = load_module()
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
for title in (
"Provider Access Healthy",
"Quota Fetch Healthy",
"Oldest Quota Sample",
):
panel = panels[title]
expression = panel["targets"][0]["expr"]
assert "last_over_time" in expression
assert "[20m]" in expression
assert "vector(-1)" in expression
mappings = panel["fieldConfig"]["defaults"]["mappings"]
assert mappings[0]["options"]["-1"]["text"] == "unavailable"
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric(): def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
"""The first three KPI bands must compare providers without layout gaps.""" """The first three KPI bands must compare providers without layout gaps."""
mod = load_module() mod = load_module()