diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index 23f6aab0..762d59bd 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5653,12 +5653,31 @@ def build_ai_dashboard(): legend_placement="right", description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", ), + stat_panel( + 30, + "Claude Quota Auth Expires In", + "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)", + {"h": 4, "w": 6, "x": 0, "y": 44}, + unit="s", + decimals=0, + instant=True, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "red", "value": None}, + {"color": "yellow", "value": 86400}, + {"color": "green", "value": 604800}, + ], + }, + description="Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not.", + ), ] for panel in panels: if panel["title"] in { "Provider Access Healthy", "Quota Fetch Healthy", "Oldest Quota Sample", + "Claude Quota Auth Expires In", }: panel["fieldConfig"]["defaults"]["mappings"] = [ { diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index abee6932..c162fe75 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded - ai.bstein.dev/config-rev: "20260824-claude-fable-quota" + ai.bstein.dev/config-rev: "20260825-claude-quota-expiry" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/scripts/ai_usage_claude.py b/services/hermes/scripts/ai_usage_claude.py index bfe00bd7..f64e7980 100644 --- a/services/hermes/scripts/ai_usage_claude.py +++ b/services/hermes/scripts/ai_usage_claude.py @@ -213,6 +213,22 @@ def _read_credentials() -> tuple[dict[str, Any], dict[str, Any]]: return document, credentials +def credential_refresh_expiry_timestamp() -> float | None: + """Return only the safe refresh-grant expiry from the private document.""" + try: + _document, credentials = _read_credentials() + except QuotaNotExposed: + return None + expires_at = credentials.get("refreshTokenExpiresAt") + if ( + isinstance(expires_at, bool) + or not isinstance(expires_at, (int, float)) + or expires_at <= 0 + ): + return None + return float(expires_at) / 1000 + + def _write_credentials(document: dict[str, Any]) -> None: """Atomically persist a provider-rotated OAuth document for Vault sync.""" encoded = (json.dumps(document, separators=(",", ":")) + "\n").encode("utf-8") diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index bb271551..5ed5bc9f 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -39,6 +39,7 @@ METRIC_HELP = { "atlas_ai_provider_authenticated": "Whether the first-party provider access boundary is authenticated.", "atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.", "atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.", + "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds": "Unix timestamp when a provider quota credential requires interactive renewal.", "atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.", "atlas_ai_quota_last_success_timestamp_seconds": "Unix timestamp of the latest successful quota fetch.", "atlas_ai_quota_remaining_percent": "Remaining percentage in a first-party coding CLI quota window.", @@ -361,6 +362,17 @@ class Collector: _provider_authenticated(provider) or quota_proves_access ) samples: list[Sample] = [] + claude_credential_expiry = ( + claude_query.credential_refresh_expiry_timestamp() + ) + if claude_credential_expiry is not None: + samples.append( + Sample( + "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds", + {"provider": "anthropic"}, + claude_credential_expiry, + ) + ) for provider, state in states.items(): labels = {"provider": provider} samples.extend(state.samples) diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index 48a3cc11..2f3c7294 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -1897,6 +1897,83 @@ } }, "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage." + }, + { + "id": 30, + "type": "stat", + "title": "Claude Quota Auth Expires In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 44 + }, + "targets": [ + { + "expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 86400 + }, + { + "color": "dark-green", + "value": 604800 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not." } ], "time": { diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index 7406f703..1eef8a22 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -1906,6 +1906,83 @@ data: } }, "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage." + }, + { + "id": 30, + "type": "stat", + "title": "Claude Quota Auth Expires In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 44 + }, + "targets": [ + { + "expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 86400 + }, + { + "color": "dark-green", + "value": 604800 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not." } ], "time": { diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index e77b7afe..0c5ec856 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -764,13 +764,19 @@ spec: group_interval: 5m repeat_interval: 2h routes: - # Triage escalations are the only alerts mailed today; everything - # else stays on the silent default receiver until it is tuned. + # Mail only actionable triage failures and advance warnings for the + # manually renewable Claude quota credential. Everything else stays + # on the silent default receiver until it is tuned. - receiver: email matchers: - service = hermes-triage group_wait: 60s repeat_interval: 12h + - receiver: email-platform + matchers: + - service = hermes-ai-quota + group_wait: 60s + repeat_interval: 24h receivers: - name: default - name: email @@ -779,3 +785,9 @@ spec: send_resolved: true headers: Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }} {{ .CommonLabels.jenkins_job }}/{{ .CommonLabels.build }}' + - name: email-platform + email_configs: + - to: brad@bstein.dev + send_resolved: true + headers: + Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }}' diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 17db8e7f..b2e6bec9 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -34,6 +34,38 @@ data: annotations: summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/" + hermes-ai-quota.yaml: | + groups: + - name: hermes.ai.quota + interval: 1m + rules: + - alert: HermesClaudeQuotaCredentialExpiring + expr: atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} - time() < 604800 + for: 15m + labels: + severity: warning + service: hermes-ai-quota + annotations: + summary: "Claude quota credential needs renewal within seven days" + description: "Authorize a new dedicated full-scope Claude login for Hermes before expiry. Inference uses a separate one-year setup token and remains available, but exact Fable weekly quota telemetry requires this credential." + - alert: HermesClaudeQuotaCredentialMissing + expr: absent(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"}) + for: 30m + labels: + severity: warning + service: hermes-ai-quota + annotations: + summary: "Claude quota credential expiry telemetry is missing" + description: "The Hermes quota exporter has not exposed the full-scope Claude credential expiry for 30 minutes. Check the dedicated Vault-backed grant before Fable telemetry disappears." + - alert: HermesClaudeFableQuotaUnavailable + expr: absent(atlas_ai_quota_remaining_percent{provider="anthropic",window="seven_day_fable"}) + for: 30m + labels: + severity: warning + service: hermes-ai-quota + annotations: + summary: "Claude Fable weekly quota telemetry is missing" + description: "The scoped Claude usage endpoint has not produced the Fable weekly quota for 30 minutes. Check the dedicated OAuth grant and exporter fetch health." atlas-availability.yaml: | groups: - name: atlas.availability.gateway @@ -411,7 +443,7 @@ spec: labels: app: vmalert-atlas-availability annotations: - bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert" + bstein.dev/rules-revision: "2026-08-25-hermes-ai-quota-expiry" spec: serviceAccountName: vmalert-atlas-availability affinity: diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index eec1f23d..2bb3ec2a 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -6,6 +6,8 @@ import importlib.util import sys from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py" @@ -49,6 +51,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert "Provider Access Healthy" in panels assert "Quota Fetch Healthy" in panels assert "Provider Selections (Range)" in panels + assert "Claude Quota Auth Expires In" in panels assert "Local Classifier Calls" in panels assert "Hermes Workload CPU (Attribution Proxy)" in panels assert "atlas_ai_quota_remaining_percent" in expressions @@ -128,6 +131,40 @@ def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes(): assert mappings[0]["options"]["-1"]["text"] == "unavailable" +def test_claude_quota_renewal_has_dashboard_and_alert_contracts(): + """The finite full-scope grant must warn before Fable telemetry expires.""" + mod = load_module() + panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} + expiry = panels["Claude Quota Auth Expires In"] + + assert "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds" in ( + expiry["targets"][0]["expr"] + ) + assert expiry["fieldConfig"]["defaults"]["mappings"][0]["options"]["-1"][ + "text" + ] == "unavailable" + + rules = next( + yaml.safe_load_all( + ( + ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + ).read_text() + ) + ) + quota_rules = yaml.safe_load(rules["data"]["hermes-ai-quota.yaml"]) + alerts = { + rule["alert"]: rule + for group in quota_rules["groups"] + for rule in group["rules"] + } + assert "HermesClaudeQuotaCredentialExpiring" in alerts + assert "HermesClaudeFableQuotaUnavailable" in alerts + assert all( + rule["labels"]["service"] == "hermes-ai-quota" + for rule in alerts.values() + ) + + def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric(): """The first three KPI bands must compare providers without layout gaps.""" mod = load_module() diff --git a/testing/tests/test_hermes_ai_usage_claude_scoped.py b/testing/tests/test_hermes_ai_usage_claude_scoped.py index d8519df7..f76e2e41 100644 --- a/testing/tests/test_hermes_ai_usage_claude_scoped.py +++ b/testing/tests/test_hermes_ai_usage_claude_scoped.py @@ -144,6 +144,20 @@ def test_scoped_query_uses_current_private_access_token(tmp_path, monkeypatch): assert requests[0].get_header("Authorization") == "Bearer scoped-access" +def test_credential_refresh_expiry_exposes_only_a_timestamp(tmp_path, monkeypatch): + mod = load_module() + credentials = tmp_path / ".credentials.json" + write_credentials(credentials, refreshTokenExpiresAt=2_000_000) + monkeypatch.setattr(mod, "CREDENTIALS_FILE", credentials) + + assert mod.credential_refresh_expiry_timestamp() == 2_000 + + write_credentials(credentials, refreshTokenExpiresAt=True) + assert mod.credential_refresh_expiry_timestamp() is None + credentials.unlink() + assert mod.credential_refresh_expiry_timestamp() is None + + def test_expired_scoped_token_refreshes_and_persists_rotation( tmp_path, monkeypatch ): diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index 8b41438e..6a310299 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -139,8 +139,15 @@ def test_claude_payload_uses_only_supported_quota_fields(): assert secret not in repr(samples) -def test_render_reports_failure_and_freshness_without_logging_credentials(): +def test_render_reports_failure_and_freshness_without_logging_credentials( + monkeypatch, +): mod = load_module() + monkeypatch.setattr( + mod.claude_query, + "credential_refresh_expiry_timestamp", + lambda: 1_800_000_000, + ) collector = mod.Collector() state = collector._providers["openai"] state.samples = [ @@ -164,6 +171,10 @@ def test_render_reports_failure_and_freshness_without_logging_credentials(): 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered ) + assert ( + 'atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} 1800000000' + in rendered + ) assert "accessToken" not in rendered assert "refreshToken" not in rendered @@ -274,7 +285,7 @@ def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module(): environment = {item["name"]: item["value"] for item in exporter["env"]} assert annotations["ai.bstein.dev/config-rev"] == ( - "20260824-claude-fable-quota" + "20260825-claude-quota-expiry" ) assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45" assert environment["ATLAS_AI_AUTHENTICATION_GRACE_SECONDS"] == "1200"