titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): preserve quota data across rollouts

Browse Source
This commit is contained in:
jenkins 2026-08-25 18:33:18 -03:00
parent 6f783b7778
commit 342677dde7
4 changed files with 150 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scripts/render/dashboards_render_atlas.py
View File

@ -5170,6 +5170,7 @@ def build_testing_dashboard():
def build_ai_dashboard():
"""Build the internal AI quota, routing, and workload dashboard."""
quota_snapshot_range = "20m"
remaining_thresholds = {
"mode": "absolute",
"steps": [
@ -5253,18 +5254,25 @@ def build_ai_dashboard():
def snapshot_fresh(provider):
"""Require authenticated access and a successful snapshot under 20m old."""
authenticated = f'atlas_ai_provider_authenticated{{provider="{provider}"}} == 1'
authenticated = (
"last_over_time(atlas_ai_provider_authenticated"
f'{{provider="{provider}"}}[{quota_snapshot_range}]) == 1'
)
last_success = (
"time() - atlas_ai_quota_last_success_timestamp_seconds"
f'{{provider="{provider}"}} < 1200'
"time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds"
f'{{provider="{provider}"}}[{quota_snapshot_range}]) < 1200'
)
return f"({authenticated}) and on(provider) ({last_success})"
openai_snapshot = snapshot_fresh("openai")
anthropic_snapshot = snapshot_fresh("anthropic")
openai_authenticated = 'atlas_ai_provider_authenticated{provider="openai"} == 1'
openai_authenticated = (
"last_over_time(atlas_ai_provider_authenticated"
f'{{provider="openai"}}[{quota_snapshot_range}]) == 1'
)
anthropic_authenticated = (
'atlas_ai_provider_authenticated{provider="anthropic"} == 1'
"last_over_time(atlas_ai_provider_authenticated"
f'{{provider="anthropic"}}[{quota_snapshot_range}]) == 1'
)
quota = "atlas_ai_quota_remaining_percent"
reset = "atlas_ai_quota_reset_timestamp_seconds"
@ -5272,8 +5280,9 @@ def build_ai_dashboard():
def provider_quota(metric, provider, window):
"""Aggregate provider quota buckets into one conservative KPI."""
return (
f'min by (provider) ({metric}{{provider="{provider}",'
f'window="{window}"}})'
"min by (provider) (last_over_time("
f'{metric}{{provider="{provider}",window="{window}"}}'
f"[{quota_snapshot_range}]))"
)
def provider_routed_tokens(provider, window):
@ -5436,7 +5445,7 @@ def build_ai_dashboard():
stat_panel(
13,
"Provider Access Healthy",
"sum(atlas_ai_provider_authenticated) or on() vector(0)",
f"sum(last_over_time(atlas_ai_provider_authenticated[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 0, "y": 8},
instant=True,
thresholds={
@ -5448,12 +5457,12 @@ def build_ai_dashboard():
],
},
value_suffix=" / 2",
description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.",
description="Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access.",
),
stat_panel(
14,
"Quota Fetch Healthy",
"sum(atlas_ai_quota_fetch_success) or on() vector(0)",
f"sum(last_over_time(atlas_ai_quota_fetch_success[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 3, "y": 8},
instant=True,
thresholds={
@ -5465,12 +5474,12 @@ def build_ai_dashboard():
],
},
value_suffix=" / 2",
description="Provider quota collectors whose latest refresh completed successfully.",
description="Provider quota collectors whose latest refresh observed within 20 minutes completed successfully.",
),
stat_panel(
15,
"Oldest Quota Sample",
"max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)",
f"max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[{quota_snapshot_range}])) or on() vector(-1)",
{"h": 4, "w": 3, "x": 6, "y": 8},
unit="s",
instant=True,
@ -5645,6 +5654,20 @@ def build_ai_dashboard():
description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.",
),
]
for panel in panels:
if panel["title"] in {
"Provider Access Healthy",
"Quota Fetch Healthy",
"Oldest Quota Sample",
}:
panel["fieldConfig"]["defaults"]["mappings"] = [
{
"type": "value",
"options": {
"-1": {"text": "unavailable", "color": "gray"},
},
}
]
return {
"uid": "atlas-ai",
"title": "Atlas AI Operations",

64
services/monitoring/dashboards/atlas-ai.json
View File

@ -20,7 +20,7 @@
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -109,7 +109,7 @@
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -186,7 +186,7 @@
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -275,7 +275,7 @@
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -478,7 +478,7 @@
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -567,7 +567,7 @@
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -644,13 +644,13 @@
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"legendFormat": "Overall weekly",
"instant": true
},
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "B",
"legendFormat": "Fable 5 weekly",
"instant": true
@ -740,7 +740,7 @@
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -943,7 +943,7 @@
},
"targets": [
{
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)",
"expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -953,7 +953,17 @@
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
@ -992,7 +1002,7 @@
},
"textMode": "value"
},
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access."
"description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access."
},
{
"id": 14,
@ -1010,7 +1020,7 @@
},
"targets": [
{
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)",
"expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -1020,7 +1030,17 @@
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
@ -1059,7 +1079,7 @@
},
"textMode": "value"
},
"description": "Provider quota collectors whose latest refresh completed successfully."
"description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully."
},
{
"id": 15,
@ -1077,7 +1097,7 @@
},
"targets": [
{
"expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)",
"expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -1087,7 +1107,17 @@
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [

64
services/monitoring/grafana-dashboard-ai.yaml
View File

@ -29,7 +29,7 @@ data:
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -118,7 +118,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -195,7 +195,7 @@ data:
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -284,7 +284,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -487,7 +487,7 @@ data:
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -576,7 +576,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -653,13 +653,13 @@ data:
},
"targets": [
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"legendFormat": "Overall weekly",
"instant": true
},
{
"expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "B",
"legendFormat": "Fable 5 weekly",
"instant": true
@ -749,7 +749,7 @@ data:
},
"targets": [
{
"expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
"expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -952,7 +952,7 @@ data:
},
"targets": [
{
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)",
"expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -962,7 +962,17 @@ data:
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
@ -1001,7 +1011,7 @@ data:
},
"textMode": "value"
},
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access."
"description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access."
},
{
"id": 14,
@ -1019,7 +1029,7 @@ data:
},
"targets": [
{
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)",
"expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -1029,7 +1039,17 @@ data:
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
@ -1068,7 +1088,7 @@ data:
},
"textMode": "value"
},
"description": "Provider quota collectors whose latest refresh completed successfully."
"description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully."
},
{
"id": 15,
@ -1086,7 +1106,7 @@ data:
},
"targets": [
{
"expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)",
"expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)",
"refId": "A",
"instant": true
}
@ -1096,7 +1116,17 @@ data:
"color": {
"mode": "thresholds"
},
"mappings": [],
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [

21
testing/tests/test_atlas_ai_dashboard.py
View File

@ -103,10 +103,31 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
):
expression = panels[title]["targets"][0]["expr"]
assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
assert "last_over_time" in expression
assert "[20m]" in expression
assert "1200" in expression
assert "atlas_ai_quota_fetch_success" not in expression
def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes():
"""Recent samples bridge rollouts, while missing telemetry stays explicit."""
mod = load_module()
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
for title in (
"Provider Access Healthy",
"Quota Fetch Healthy",
"Oldest Quota Sample",
):
panel = panels[title]
expression = panel["targets"][0]["expr"]
assert "last_over_time" in expression
assert "[20m]" in expression
assert "vector(-1)" in expression
mappings = panel["fieldConfig"]["defaults"]["mappings"]
assert mappings[0]["options"]["-1"]["text"] == "unavailable"
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
"""The first three KPI bands must compare providers without layout gaps."""
mod = load_module()