From 342677dde77b50829c310563bc4e4c42830fe6dd Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 18:33:18 -0300 Subject: [PATCH] monitoring(ai): preserve quota data across rollouts --- scripts/render/dashboards_render_atlas.py | 47 ++++++++++---- services/monitoring/dashboards/atlas-ai.json | 64 ++++++++++++++----- services/monitoring/grafana-dashboard-ai.yaml | 64 ++++++++++++++----- testing/tests/test_atlas_ai_dashboard.py | 21 ++++++ 4 files changed, 150 insertions(+), 46 deletions(-) diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index 6541f7c5..23f6aab0 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5170,6 +5170,7 @@ def build_testing_dashboard(): def build_ai_dashboard(): """Build the internal AI quota, routing, and workload dashboard.""" + quota_snapshot_range = "20m" remaining_thresholds = { "mode": "absolute", "steps": [ @@ -5253,18 +5254,25 @@ def build_ai_dashboard(): def snapshot_fresh(provider): """Require authenticated access and a successful snapshot under 20m old.""" - authenticated = f'atlas_ai_provider_authenticated{{provider="{provider}"}} == 1' + authenticated = ( + "last_over_time(atlas_ai_provider_authenticated" + f'{{provider="{provider}"}}[{quota_snapshot_range}]) == 1' + ) last_success = ( - "time() - atlas_ai_quota_last_success_timestamp_seconds" - f'{{provider="{provider}"}} < 1200' + "time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds" + f'{{provider="{provider}"}}[{quota_snapshot_range}]) < 1200' ) return f"({authenticated}) and on(provider) ({last_success})" openai_snapshot = snapshot_fresh("openai") anthropic_snapshot = snapshot_fresh("anthropic") - openai_authenticated = 'atlas_ai_provider_authenticated{provider="openai"} == 1' + openai_authenticated = ( + "last_over_time(atlas_ai_provider_authenticated" + f'{{provider="openai"}}[{quota_snapshot_range}]) == 1' + ) anthropic_authenticated = ( - 'atlas_ai_provider_authenticated{provider="anthropic"} == 1' + "last_over_time(atlas_ai_provider_authenticated" + f'{{provider="anthropic"}}[{quota_snapshot_range}]) == 1' ) quota = "atlas_ai_quota_remaining_percent" reset = "atlas_ai_quota_reset_timestamp_seconds" @@ -5272,8 +5280,9 @@ def build_ai_dashboard(): def provider_quota(metric, provider, window): """Aggregate provider quota buckets into one conservative KPI.""" return ( - f'min by (provider) ({metric}{{provider="{provider}",' - f'window="{window}"}})' + "min by (provider) (last_over_time(" + f'{metric}{{provider="{provider}",window="{window}"}}' + f"[{quota_snapshot_range}]))" ) def provider_routed_tokens(provider, window): @@ -5436,7 +5445,7 @@ def build_ai_dashboard(): stat_panel( 13, "Provider Access Healthy", - "sum(atlas_ai_provider_authenticated) or on() vector(0)", + f"sum(last_over_time(atlas_ai_provider_authenticated[{quota_snapshot_range}])) or on() vector(-1)", {"h": 4, "w": 3, "x": 0, "y": 8}, instant=True, thresholds={ @@ -5448,12 +5457,12 @@ def build_ai_dashboard(): ], }, value_suffix=" / 2", - description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.", + description="Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access.", ), stat_panel( 14, "Quota Fetch Healthy", - "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + f"sum(last_over_time(atlas_ai_quota_fetch_success[{quota_snapshot_range}])) or on() vector(-1)", {"h": 4, "w": 3, "x": 3, "y": 8}, instant=True, thresholds={ @@ -5465,12 +5474,12 @@ def build_ai_dashboard(): ], }, value_suffix=" / 2", - description="Provider quota collectors whose latest refresh completed successfully.", + description="Provider quota collectors whose latest refresh observed within 20 minutes completed successfully.", ), stat_panel( 15, "Oldest Quota Sample", - "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + f"max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[{quota_snapshot_range}])) or on() vector(-1)", {"h": 4, "w": 3, "x": 6, "y": 8}, unit="s", instant=True, @@ -5645,6 +5654,20 @@ def build_ai_dashboard(): description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", ), ] + for panel in panels: + if panel["title"] in { + "Provider Access Healthy", + "Quota Fetch Healthy", + "Oldest Quota Sample", + }: + panel["fieldConfig"]["defaults"]["mappings"] = [ + { + "type": "value", + "options": { + "-1": {"text": "unavailable", "color": "gray"}, + }, + } + ] return { "uid": "atlas-ai", "title": "Atlas AI Operations", diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index 862e0b65..48a3cc11 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -109,7 +109,7 @@ }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -186,7 +186,7 @@ }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -275,7 +275,7 @@ }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -478,7 +478,7 @@ }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -567,7 +567,7 @@ }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -644,13 +644,13 @@ }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "legendFormat": "Overall weekly", "instant": true }, { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "B", "legendFormat": "Fable 5 weekly", "instant": true @@ -740,7 +740,7 @@ }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -943,7 +943,7 @@ }, "targets": [ { - "expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", + "expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -953,7 +953,17 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -992,7 +1002,7 @@ }, "textMode": "value" }, - "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." + "description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access." }, { "id": 14, @@ -1010,7 +1020,7 @@ }, "targets": [ { - "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -1020,7 +1030,17 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -1059,7 +1079,7 @@ }, "textMode": "value" }, - "description": "Provider quota collectors whose latest refresh completed successfully." + "description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully." }, { "id": 15, @@ -1077,7 +1097,7 @@ }, "targets": [ { - "expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + "expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -1087,7 +1107,17 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index b96f13d4..7406f703 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -118,7 +118,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -195,7 +195,7 @@ data: }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"openai\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -284,7 +284,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"openai\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -487,7 +487,7 @@ data: }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -576,7 +576,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -653,13 +653,13 @@ data: }, "targets": [ { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "legendFormat": "Overall weekly", "instant": true }, { - "expr": "((min by (provider) (atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((min by (provider) (last_over_time(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day_fable\"}[20m])) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "B", "legendFormat": "Fable 5 weekly", "instant": true @@ -749,7 +749,7 @@ data: }, "targets": [ { - "expr": "((clamp_min(min by (provider) (atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}) - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((clamp_min(min by (provider) (last_over_time(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"}[20m])) - time(), 0) and on(provider) ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) and on(provider) (time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"}[20m]) < 1200))) or on() ((last_over_time(atlas_ai_provider_authenticated{provider=\"anthropic\"}[20m]) == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -952,7 +952,7 @@ data: }, "targets": [ { - "expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", + "expr": "sum(last_over_time(atlas_ai_provider_authenticated[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -962,7 +962,17 @@ data: "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -1001,7 +1011,7 @@ data: }, "textMode": "value" }, - "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." + "description": "Authenticated first-party provider boundaries observed within 20 minutes. Quota visibility is tracked separately from model access." }, { "id": 14, @@ -1019,7 +1029,7 @@ data: }, "targets": [ { - "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "expr": "sum(last_over_time(atlas_ai_quota_fetch_success[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -1029,7 +1039,17 @@ data: "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -1068,7 +1088,7 @@ data: }, "textMode": "value" }, - "description": "Provider quota collectors whose latest refresh completed successfully." + "description": "Provider quota collectors whose latest refresh observed within 20 minutes completed successfully." }, { "id": 15, @@ -1086,7 +1106,7 @@ data: }, "targets": [ { - "expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + "expr": "max(time() - last_over_time(atlas_ai_quota_last_success_timestamp_seconds[20m])) or on() vector(-1)", "refId": "A", "instant": true } @@ -1096,7 +1116,17 @@ data: "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index e20fcfb5..eec1f23d 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -103,10 +103,31 @@ def test_ai_quota_panels_retain_a_bounded_last_good_snapshot(): ): expression = panels[title]["targets"][0]["expr"] assert "atlas_ai_quota_last_success_timestamp_seconds" in expression + assert "last_over_time" in expression + assert "[20m]" in expression assert "1200" in expression assert "atlas_ai_quota_fetch_success" not in expression +def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes(): + """Recent samples bridge rollouts, while missing telemetry stays explicit.""" + mod = load_module() + panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} + + for title in ( + "Provider Access Healthy", + "Quota Fetch Healthy", + "Oldest Quota Sample", + ): + panel = panels[title] + expression = panel["targets"][0]["expr"] + assert "last_over_time" in expression + assert "[20m]" in expression + assert "vector(-1)" in expression + mappings = panel["fieldConfig"]["defaults"]["mappings"] + assert mappings[0]["options"]["-1"]["text"] == "unavailable" + + def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric(): """The first three KPI bands must compare providers without layout gaps.""" mod = load_module()