monitoring(ai): alert before Claude quota auth expires

scripts/render/dashboards_render_atlas.py

@@ -5653,12 +5653,31 @@ def build_ai_dashboard():
             legend_placement="right",
             description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.",
         ),
+        stat_panel(
+            30,
+            "Claude Quota Auth Expires In",
+            "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
+            {"h": 4, "w": 6, "x": 0, "y": 44},
+            unit="s",
+            decimals=0,
+            instant=True,
+            thresholds={
+                "mode": "absolute",
+                "steps": [
+                    {"color": "red", "value": None},
+                    {"color": "yellow", "value": 86400},
+                    {"color": "green", "value": 604800},
+                ],
+            },
+            description="Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not.",
+        ),
     ]
     for panel in panels:
         if panel["title"] in {
             "Provider Access Healthy",
             "Quota Fetch Healthy",
             "Oldest Quota Sample",
+            "Claude Quota Auth Expires In",
         }:
             panel["fieldConfig"]["defaults"]["mappings"] = [
                 {

services/hermes/agent-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded
-        ai.bstein.dev/config-rev: "20260824-claude-fable-quota"
+        ai.bstein.dev/config-rev: "20260825-claude-quota-expiry"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"

services/hermes/scripts/ai_usage_claude.py

@@ -213,6 +213,22 @@ def _read_credentials() -> tuple[dict[str, Any], dict[str, Any]]:
     return document, credentials
 
 
+def credential_refresh_expiry_timestamp() -> float | None:
+    """Return only the safe refresh-grant expiry from the private document."""
+    try:
+        _document, credentials = _read_credentials()
+    except QuotaNotExposed:
+        return None
+    expires_at = credentials.get("refreshTokenExpiresAt")
+    if (
+        isinstance(expires_at, bool)
+        or not isinstance(expires_at, (int, float))
+        or expires_at <= 0
+    ):
+        return None
+    return float(expires_at) / 1000
+
+
 def _write_credentials(document: dict[str, Any]) -> None:
     """Atomically persist a provider-rotated OAuth document for Vault sync."""
     encoded = (json.dumps(document, separators=(",", ":")) + "\n").encode("utf-8")

services/hermes/scripts/ai_usage_exporter.py

@@ -39,6 +39,7 @@ METRIC_HELP = {
     "atlas_ai_provider_authenticated": "Whether the first-party provider access boundary is authenticated.",
     "atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.",
     "atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.",
+    "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds": "Unix timestamp when a provider quota credential requires interactive renewal.",
     "atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.",
     "atlas_ai_quota_last_success_timestamp_seconds": "Unix timestamp of the latest successful quota fetch.",
     "atlas_ai_quota_remaining_percent": "Remaining percentage in a first-party coding CLI quota window.",
@@ -361,6 +362,17 @@ class Collector:
                 _provider_authenticated(provider) or quota_proves_access
             )
         samples: list[Sample] = []
+        claude_credential_expiry = (
+            claude_query.credential_refresh_expiry_timestamp()
+        )
+        if claude_credential_expiry is not None:
+            samples.append(
+                Sample(
+                    "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds",
+                    {"provider": "anthropic"},
+                    claude_credential_expiry,
+                )
+            )
         for provider, state in states.items():
             labels = {"provider": provider}
             samples.extend(state.samples)

services/monitoring/dashboards/atlas-ai.json

@@ -1897,6 +1897,83 @@
         }
       },
       "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage."
+    },
+    {
+      "id": 30,
+      "type": "stat",
+      "title": "Claude Quota Auth Expires In",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "atlas-vm"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 44
+      },
+      "targets": [
+        {
+          "expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "-1": {
+                  "text": "unavailable",
+                  "color": "gray"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "dark-red",
+                "value": null
+              },
+              {
+                "color": "dark-yellow",
+                "value": 86400
+              },
+              {
+                "color": "dark-green",
+                "value": 604800
+              }
+            ]
+          },
+          "unit": "s",
+          "custom": {
+            "displayMode": "auto"
+          },
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      },
+      "description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not."
     }
   ],
   "time": {

services/monitoring/grafana-dashboard-ai.yaml

@@ -1906,6 +1906,83 @@ data:
             }
           },
           "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage."
+        },
+        {
+          "id": 30,
+          "type": "stat",
+          "title": "Claude Quota Auth Expires In",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "atlas-vm"
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 6,
+            "x": 0,
+            "y": 44
+          },
+          "targets": [
+            {
+              "expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
+              "refId": "A",
+              "instant": true
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [
+                {
+                  "type": "value",
+                  "options": {
+                    "-1": {
+                      "text": "unavailable",
+                      "color": "gray"
+                    }
+                  }
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "dark-red",
+                    "value": null
+                  },
+                  {
+                    "color": "dark-yellow",
+                    "value": 86400
+                  },
+                  {
+                    "color": "dark-green",
+                    "value": 604800
+                  }
+                ]
+              },
+              "unit": "s",
+              "custom": {
+                "displayMode": "auto"
+              },
+              "decimals": 0
+            },
+            "overrides": []
+          },
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "textMode": "value"
+          },
+          "description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not."
         }
       ],
       "time": {

services/monitoring/helmrelease.yaml

@@ -764,13 +764,19 @@ spec:
         group_interval: 5m
         repeat_interval: 2h
         routes:
-          # Triage escalations are the only alerts mailed today; everything
+          # Mail only actionable triage failures and advance warnings for the
-          # else stays on the silent default receiver until it is tuned.
+          # manually renewable Claude quota credential. Everything else stays
+          # on the silent default receiver until it is tuned.
           - receiver: email
             matchers:
               - service = hermes-triage
             group_wait: 60s
             repeat_interval: 12h
+          - receiver: email-platform
+            matchers:
+              - service = hermes-ai-quota
+            group_wait: 60s
+            repeat_interval: 24h
       receivers:
         - name: default
         - name: email
@@ -779,3 +785,9 @@ spec:
               send_resolved: true
               headers:
                 Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }} {{ .CommonLabels.jenkins_job }}/{{ .CommonLabels.build }}'
+        - name: email-platform
+          email_configs:
+            - to: brad@bstein.dev
+              send_resolved: true
+              headers:
+                Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }}'

services/monitoring/vmalert-atlas-availability.yaml

@@ -34,6 +34,38 @@ data:
             annotations:
               summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
               description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
+  hermes-ai-quota.yaml: |
+    groups:
+      - name: hermes.ai.quota
+        interval: 1m
+        rules:
+          - alert: HermesClaudeQuotaCredentialExpiring
+            expr: atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} - time() < 604800
+            for: 15m
+            labels:
+              severity: warning
+              service: hermes-ai-quota
+            annotations:
+              summary: "Claude quota credential needs renewal within seven days"
+              description: "Authorize a new dedicated full-scope Claude login for Hermes before expiry. Inference uses a separate one-year setup token and remains available, but exact Fable weekly quota telemetry requires this credential."
+          - alert: HermesClaudeQuotaCredentialMissing
+            expr: absent(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"})
+            for: 30m
+            labels:
+              severity: warning
+              service: hermes-ai-quota
+            annotations:
+              summary: "Claude quota credential expiry telemetry is missing"
+              description: "The Hermes quota exporter has not exposed the full-scope Claude credential expiry for 30 minutes. Check the dedicated Vault-backed grant before Fable telemetry disappears."
+          - alert: HermesClaudeFableQuotaUnavailable
+            expr: absent(atlas_ai_quota_remaining_percent{provider="anthropic",window="seven_day_fable"})
+            for: 30m
+            labels:
+              severity: warning
+              service: hermes-ai-quota
+            annotations:
+              summary: "Claude Fable weekly quota telemetry is missing"
+              description: "The scoped Claude usage endpoint has not produced the Fable weekly quota for 30 minutes. Check the dedicated OAuth grant and exporter fetch health."
   atlas-availability.yaml: |
     groups:
       - name: atlas.availability.gateway
@@ -411,7 +443,7 @@ spec:
       labels:
         app: vmalert-atlas-availability
       annotations:
-        bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
+        bstein.dev/rules-revision: "2026-08-25-hermes-ai-quota-expiry"
     spec:
       serviceAccountName: vmalert-atlas-availability
       affinity:

testing/tests/test_atlas_ai_dashboard.py

@@ -6,6 +6,8 @@ import importlib.util
 import sys
 from pathlib import Path
 
+import yaml
+
 
 ROOT = Path(__file__).resolve().parents[2]
 SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py"
@@ -49,6 +51,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
     assert "Provider Access Healthy" in panels
     assert "Quota Fetch Healthy" in panels
     assert "Provider Selections (Range)" in panels
+    assert "Claude Quota Auth Expires In" in panels
     assert "Local Classifier Calls" in panels
     assert "Hermes Workload CPU (Attribution Proxy)" in panels
     assert "atlas_ai_quota_remaining_percent" in expressions
@@ -128,6 +131,40 @@ def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes():
         assert mappings[0]["options"]["-1"]["text"] == "unavailable"
 
 
+def test_claude_quota_renewal_has_dashboard_and_alert_contracts():
+    """The finite full-scope grant must warn before Fable telemetry expires."""
+    mod = load_module()
+    panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
+    expiry = panels["Claude Quota Auth Expires In"]
+
+    assert "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds" in (
+        expiry["targets"][0]["expr"]
+    )
+    assert expiry["fieldConfig"]["defaults"]["mappings"][0]["options"]["-1"][
+        "text"
+    ] == "unavailable"
+
+    rules = next(
+        yaml.safe_load_all(
+            (
+                ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
+            ).read_text()
+        )
+    )
+    quota_rules = yaml.safe_load(rules["data"]["hermes-ai-quota.yaml"])
+    alerts = {
+        rule["alert"]: rule
+        for group in quota_rules["groups"]
+        for rule in group["rules"]
+    }
+    assert "HermesClaudeQuotaCredentialExpiring" in alerts
+    assert "HermesClaudeFableQuotaUnavailable" in alerts
+    assert all(
+        rule["labels"]["service"] == "hermes-ai-quota"
+        for rule in alerts.values()
+    )
+
+
 def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
     """The first three KPI bands must compare providers without layout gaps."""
     mod = load_module()

testing/tests/test_hermes_ai_usage_claude_scoped.py

@@ -144,6 +144,20 @@ def test_scoped_query_uses_current_private_access_token(tmp_path, monkeypatch):
     assert requests[0].get_header("Authorization") == "Bearer scoped-access"
 
 
+def test_credential_refresh_expiry_exposes_only_a_timestamp(tmp_path, monkeypatch):
+    mod = load_module()
+    credentials = tmp_path / ".credentials.json"
+    write_credentials(credentials, refreshTokenExpiresAt=2_000_000)
+    monkeypatch.setattr(mod, "CREDENTIALS_FILE", credentials)
+
+    assert mod.credential_refresh_expiry_timestamp() == 2_000
+
+    write_credentials(credentials, refreshTokenExpiresAt=True)
+    assert mod.credential_refresh_expiry_timestamp() is None
+    credentials.unlink()
+    assert mod.credential_refresh_expiry_timestamp() is None
+
+
 def test_expired_scoped_token_refreshes_and_persists_rotation(
     tmp_path, monkeypatch
 ):

testing/tests/test_hermes_ai_usage_exporter.py

@@ -139,8 +139,15 @@ def test_claude_payload_uses_only_supported_quota_fields():
     assert secret not in repr(samples)
 
 
-def test_render_reports_failure_and_freshness_without_logging_credentials():
+def test_render_reports_failure_and_freshness_without_logging_credentials(
+    monkeypatch,
+):
     mod = load_module()
+    monkeypatch.setattr(
+        mod.claude_query,
+        "credential_refresh_expiry_timestamp",
+        lambda: 1_800_000_000,
+    )
     collector = mod.Collector()
     state = collector._providers["openai"]
     state.samples = [
@@ -164,6 +171,10 @@ def test_render_reports_failure_and_freshness_without_logging_credentials():
         'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42'
         in rendered
     )
+    assert (
+        'atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} 1800000000'
+        in rendered
+    )
     assert "accessToken" not in rendered
     assert "refreshToken" not in rendered
 
@@ -274,7 +285,7 @@ def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module():
     environment = {item["name"]: item["value"] for item in exporter["env"]}
 
     assert annotations["ai.bstein.dev/config-rev"] == (
-        "20260824-claude-fable-quota"
+        "20260825-claude-quota-expiry"
     )
     assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45"
     assert environment["ATLAS_AI_AUTHENTICATION_GRACE_SECONDS"] == "1200"