monitoring(ai): measure Claude routed usage

scripts/render/dashboards_render_atlas.py

@@ -5231,6 +5231,22 @@ def build_ai_dashboard():
     )
     quota = "atlas_ai_quota_remaining_percent"
     reset = "atlas_ai_quota_reset_timestamp_seconds"
+    claude_models = 'model=~"route/claude/.+"'
+
+    def claude_routed_tokens(window):
+        """Count the disjoint Claude token classes Switchyard observed."""
+        metrics = (
+            "switchyard_prompt_tokens_total",
+            "switchyard_cached_tokens_total",
+            "switchyard_cache_creation_tokens_total",
+            "switchyard_completion_tokens_total",
+        )
+        terms = [
+            f'(sum(increase({metric}{{{claude_models}}}[{window}])) or on() vector(0))'
+            for metric in metrics
+        ]
+        return " + ".join(terms)
+
     panels = [
         quota_stat(
             1,
@@ -5300,21 +5316,25 @@ def build_ai_dashboard():
             {"h": 4, "w": 4, "x": 0, "y": 4},
             unit="s",
         ),
-        quota_stat(
+        stat_panel(
             8,
-            "Claude 5h Reset In",
+            "Claude Routed Tokens (24h)",
-            f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})',
+            claude_routed_tokens("24h"),
             {"h": 4, "w": 4, "x": 4, "y": 4},
-            unit="s",
+            unit="short",
-            authenticated_expr=anthropic_authenticated,
+            decimals=0,
+            instant=True,
+            description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance.",
         ),
-        quota_stat(
+        stat_panel(
             9,
-            "Claude 7d Reset In",
+            "Claude Routed Tokens (7d)",
-            f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})',
+            claude_routed_tokens("7d"),
             {"h": 4, "w": 4, "x": 8, "y": 4},
-            unit="s",
+            unit="short",
-            authenticated_expr=anthropic_authenticated,
+            decimals=0,
+            instant=True,
+            description="Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope.",
         ),
         quota_stat(
             10,

services/monitoring/dashboards/atlas-ai.json

@@ -577,7 +577,7 @@
     {
       "id": 8,
       "type": "stat",
-      "title": "Claude 5h Reset In",
+      "title": "Claude Routed Tokens (24h)",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -590,7 +590,7 @@
       },
       "targets": [
         {
-          "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+          "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))",
           "refId": "A",
           "instant": true
         }
@@ -600,39 +600,25 @@
           "color": {
             "mode": "thresholds"
           },
-          "mappings": [
+          "mappings": [],
-            {
-              "type": "value",
-              "options": {
-                "-2": {
-                  "text": "not exposed",
-                  "color": "dark-blue"
-                },
-                "-1": {
-                  "text": "unavailable",
-                  "color": "gray"
-                }
-              }
-            }
-          ],
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "gray",
+                "color": "rgba(115, 115, 115, 1)",
                 "value": null
               },
               {
-                "color": "dark-blue",
+                "color": "dark-green",
-                "value": 0
+                "value": 1
               }
             ]
           },
-          "unit": "s",
+          "unit": "short",
           "custom": {
             "displayMode": "auto"
           },
-          "decimals": 1
+          "decimals": 0
         },
         "overrides": []
       },
@@ -649,12 +635,12 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance."
     },
     {
       "id": 9,
       "type": "stat",
-      "title": "Claude 7d Reset In",
+      "title": "Claude Routed Tokens (7d)",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -667,7 +653,7 @@
       },
       "targets": [
         {
-          "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+          "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))",
           "refId": "A",
           "instant": true
         }
@@ -677,39 +663,25 @@
           "color": {
             "mode": "thresholds"
           },
-          "mappings": [
+          "mappings": [],
-            {
-              "type": "value",
-              "options": {
-                "-2": {
-                  "text": "not exposed",
-                  "color": "dark-blue"
-                },
-                "-1": {
-                  "text": "unavailable",
-                  "color": "gray"
-                }
-              }
-            }
-          ],
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "gray",
+                "color": "rgba(115, 115, 115, 1)",
                 "value": null
               },
               {
-                "color": "dark-blue",
+                "color": "dark-green",
-                "value": 0
+                "value": 1
               }
             ]
           },
-          "unit": "s",
+          "unit": "short",
           "custom": {
             "displayMode": "auto"
           },
-          "decimals": 1
+          "decimals": 0
         },
         "overrides": []
       },
@@ -726,7 +698,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope."
     },
     {
       "id": 10,

services/monitoring/grafana-dashboard-ai.yaml

@@ -586,7 +586,7 @@ data:
         {
           "id": 8,
           "type": "stat",
-          "title": "Claude 5h Reset In",
+          "title": "Claude Routed Tokens (24h)",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -599,7 +599,7 @@ data:
           },
           "targets": [
             {
-              "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+              "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[24h])) or on() vector(0))",
               "refId": "A",
               "instant": true
             }
@@ -609,39 +609,25 @@ data:
               "color": {
                 "mode": "thresholds"
               },
-              "mappings": [
+              "mappings": [],
-                {
-                  "type": "value",
-                  "options": {
-                    "-2": {
-                      "text": "not exposed",
-                      "color": "dark-blue"
-                    },
-                    "-1": {
-                      "text": "unavailable",
-                      "color": "gray"
-                    }
-                  }
-                }
-              ],
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "gray",
+                    "color": "rgba(115, 115, 115, 1)",
                     "value": null
                   },
                   {
-                    "color": "dark-blue",
+                    "color": "dark-green",
-                    "value": 0
+                    "value": 1
                   }
                 ]
               },
-              "unit": "s",
+              "unit": "short",
               "custom": {
                 "displayMode": "auto"
               },
-              "decimals": 1
+              "decimals": 0
             },
             "overrides": []
           },
@@ -658,12 +644,12 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last 24 hours. This measures consumption even when Anthropic does not expose the subscription allowance."
         },
         {
           "id": 9,
           "type": "stat",
-          "title": "Claude 7d Reset In",
+          "title": "Claude Routed Tokens (7d)",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -676,7 +662,7 @@ data:
           },
           "targets": [
             {
-              "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+              "expr": "(sum(increase(switchyard_prompt_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cached_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_cache_creation_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0)) + (sum(increase(switchyard_completion_tokens_total{model=~\"route/claude/.+\"}[7d])) or on() vector(0))",
               "refId": "A",
               "instant": true
             }
@@ -686,39 +672,25 @@ data:
               "color": {
                 "mode": "thresholds"
               },
-              "mappings": [
+              "mappings": [],
-                {
-                  "type": "value",
-                  "options": {
-                    "-2": {
-                      "text": "not exposed",
-                      "color": "dark-blue"
-                    },
-                    "-1": {
-                      "text": "unavailable",
-                      "color": "gray"
-                    }
-                  }
-                }
-              ],
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "gray",
+                    "color": "rgba(115, 115, 115, 1)",
                     "value": null
                   },
                   {
-                    "color": "dark-blue",
+                    "color": "dark-green",
-                    "value": 0
+                    "value": 1
                   }
                 ]
               },
-              "unit": "s",
+              "unit": "short",
               "custom": {
                 "displayMode": "auto"
               },
-              "decimals": 1
+              "decimals": 0
             },
             "overrides": []
           },
@@ -735,7 +707,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Prompt, cache-read, cache-creation, and completion tokens from Claude calls routed through Switchyard in the last seven days. This is measured locally and does not require a broader Anthropic OAuth scope."
         },
         {
           "id": 10,

testing/tests/test_atlas_ai_dashboard.py

@@ -37,6 +37,8 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
     assert "Codex Weekly Remaining" in panels
     assert "Claude 5h Remaining" in panels
     assert "Claude 7d Remaining" in panels
+    assert "Claude Routed Tokens (24h)" in panels
+    assert "Claude Routed Tokens (7d)" in panels
     assert "Provider Access Healthy" in panels
     assert "Provider Selections (Range)" in panels
     assert "Local Classifier Calls" in panels
@@ -46,6 +48,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
     assert "atlas_ai_provider_authenticated" in expressions
     assert "switchyard_decisions_total" in expressions
     assert "switchyard_cached_tokens_total" in expressions
+    assert 'model=~"route/claude/.+"' in expressions
     assert "switchyard_model_call_latency_ms_bucket" in expressions
     assert "/status" not in expressions
     assert all(