titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): alert before Claude quota auth expires

Browse Source
This commit is contained in:
jenkins 2026-08-25 19:15:58 -03:00
parent 2a11c8d207
commit db446c9244
11 changed files with 313 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/render/dashboards_render_atlas.py
View File

@ -5653,12 +5653,31 @@ def build_ai_dashboard():
legend_placement="right",
description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.",
),
stat_panel(
30,
"Claude Quota Auth Expires In",
"clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
{"h": 4, "w": 6, "x": 0, "y": 44},
unit="s",
decimals=0,
instant=True,
thresholds={
"mode": "absolute",
"steps": [
{"color": "red", "value": None},
{"color": "yellow", "value": 86400},
{"color": "green", "value": 604800},
],
},
description="Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not.",
),
]
for panel in panels:
if panel["title"] in {
"Provider Access Healthy",
"Quota Fetch Healthy",
"Oldest Quota Sample",
"Claude Quota Auth Expires In",
}:
panel["fieldConfig"]["defaults"]["mappings"] = [
{

2
services/hermes/agent-deployment.yaml
View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded
ai.bstein.dev/config-rev: "20260824-claude-fable-quota"
ai.bstein.dev/config-rev: "20260825-claude-quota-expiry"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"

16
services/hermes/scripts/ai_usage_claude.py
View File

@ -213,6 +213,22 @@ def _read_credentials() -> tuple[dict[str, Any], dict[str, Any]]:
return document, credentials
def credential_refresh_expiry_timestamp() -> float | None:
"""Return only the safe refresh-grant expiry from the private document."""
try:
_document, credentials = _read_credentials()
except QuotaNotExposed:
return None
expires_at = credentials.get("refreshTokenExpiresAt")
if (
isinstance(expires_at, bool)
or not isinstance(expires_at, (int, float))
or expires_at <= 0
):
return None
return float(expires_at) / 1000
def _write_credentials(document: dict[str, Any]) -> None:
"""Atomically persist a provider-rotated OAuth document for Vault sync."""
encoded = (json.dumps(document, separators=(",", ":")) + "\n").encode("utf-8")

12
services/hermes/scripts/ai_usage_exporter.py
View File

@ -39,6 +39,7 @@ METRIC_HELP = {
"atlas_ai_provider_authenticated": "Whether the first-party provider access boundary is authenticated.",
"atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.",
"atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.",
"atlas_ai_quota_credential_refresh_expiry_timestamp_seconds": "Unix timestamp when a provider quota credential requires interactive renewal.",
"atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.",
"atlas_ai_quota_last_success_timestamp_seconds": "Unix timestamp of the latest successful quota fetch.",
"atlas_ai_quota_remaining_percent": "Remaining percentage in a first-party coding CLI quota window.",
@ -361,6 +362,17 @@ class Collector:
_provider_authenticated(provider) or quota_proves_access
)
samples: list[Sample] = []
claude_credential_expiry = (
claude_query.credential_refresh_expiry_timestamp()
)
if claude_credential_expiry is not None:
samples.append(
Sample(
"atlas_ai_quota_credential_refresh_expiry_timestamp_seconds",
{"provider": "anthropic"},
claude_credential_expiry,
)
)
for provider, state in states.items():
labels = {"provider": provider}
samples.extend(state.samples)

77
services/monitoring/dashboards/atlas-ai.json
View File

@ -1897,6 +1897,83 @@
}
},
"description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage."
},
{
"id": 30,
"type": "stat",
"title": "Claude Quota Auth Expires In",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 44
},
"targets": [
{
"expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "dark-red",
"value": null
},
{
"color": "dark-yellow",
"value": 86400
},
{
"color": "dark-green",
"value": 604800
}
]
},
"unit": "s",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not."
}
],
"time": {

77
services/monitoring/grafana-dashboard-ai.yaml
View File

@ -1906,6 +1906,83 @@ data:
}
},
"description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage."
},
{
"id": 30,
"type": "stat",
"title": "Claude Quota Auth Expires In",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 44
},
"targets": [
{
"expr": "clamp_min(last_over_time(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider=\"anthropic\"}[20m]) - time(), 0) or on() vector(-1)",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"-1": {
"text": "unavailable",
"color": "gray"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "dark-red",
"value": null
},
{
"color": "dark-yellow",
"value": 86400
},
{
"color": "dark-green",
"value": 604800
}
]
},
"unit": "s",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"description": "Time until the full-scope Claude quota credential needs interactive renewal. Warning begins at seven days; inference continues through its separate one-year setup token if this expires, but scoped Fable quota telemetry does not."
}
],
"time": {

16
services/monitoring/helmrelease.yaml
View File

@ -764,13 +764,19 @@ spec:
group_interval: 5m
repeat_interval: 2h
routes:
# Triage escalations are the only alerts mailed today; everything
# else stays on the silent default receiver until it is tuned.
# Mail only actionable triage failures and advance warnings for the
# manually renewable Claude quota credential. Everything else stays
# on the silent default receiver until it is tuned.
- receiver: email
matchers:
- service = hermes-triage
group_wait: 60s
repeat_interval: 12h
- receiver: email-platform
matchers:
- service = hermes-ai-quota
group_wait: 60s
repeat_interval: 24h
receivers:
- name: default
- name: email
@ -779,3 +785,9 @@ spec:
send_resolved: true
headers:
Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }} {{ .CommonLabels.jenkins_job }}/{{ .CommonLabels.build }}'
- name: email-platform
email_configs:
- to: brad@bstein.dev
send_resolved: true
headers:
Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }}'

34
services/monitoring/vmalert-atlas-availability.yaml
View File

@ -34,6 +34,38 @@ data:
annotations:
summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
hermes-ai-quota.yaml: |
groups:
- name: hermes.ai.quota
interval: 1m
rules:
- alert: HermesClaudeQuotaCredentialExpiring
expr: atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} - time() < 604800
for: 15m
labels:
severity: warning
service: hermes-ai-quota
annotations:
summary: "Claude quota credential needs renewal within seven days"
description: "Authorize a new dedicated full-scope Claude login for Hermes before expiry. Inference uses a separate one-year setup token and remains available, but exact Fable weekly quota telemetry requires this credential."
- alert: HermesClaudeQuotaCredentialMissing
expr: absent(atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"})
for: 30m
labels:
severity: warning
service: hermes-ai-quota
annotations:
summary: "Claude quota credential expiry telemetry is missing"
description: "The Hermes quota exporter has not exposed the full-scope Claude credential expiry for 30 minutes. Check the dedicated Vault-backed grant before Fable telemetry disappears."
- alert: HermesClaudeFableQuotaUnavailable
expr: absent(atlas_ai_quota_remaining_percent{provider="anthropic",window="seven_day_fable"})
for: 30m
labels:
severity: warning
service: hermes-ai-quota
annotations:
summary: "Claude Fable weekly quota telemetry is missing"
description: "The scoped Claude usage endpoint has not produced the Fable weekly quota for 30 minutes. Check the dedicated OAuth grant and exporter fetch health."
atlas-availability.yaml: |
groups:
- name: atlas.availability.gateway
@ -411,7 +443,7 @@ spec:
labels:
app: vmalert-atlas-availability
annotations:
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
bstein.dev/rules-revision: "2026-08-25-hermes-ai-quota-expiry"
spec:
serviceAccountName: vmalert-atlas-availability
affinity:

37
testing/tests/test_atlas_ai_dashboard.py
View File

@ -6,6 +6,8 @@ import importlib.util
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py"
@ -49,6 +51,7 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
assert "Provider Access Healthy" in panels
assert "Quota Fetch Healthy" in panels
assert "Provider Selections (Range)" in panels
assert "Claude Quota Auth Expires In" in panels
assert "Local Classifier Calls" in panels
assert "Hermes Workload CPU (Attribution Proxy)" in panels
assert "atlas_ai_quota_remaining_percent" in expressions
@ -128,6 +131,40 @@ def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes():
assert mappings[0]["options"]["-1"]["text"] == "unavailable"
def test_claude_quota_renewal_has_dashboard_and_alert_contracts():
"""The finite full-scope grant must warn before Fable telemetry expires."""
mod = load_module()
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
expiry = panels["Claude Quota Auth Expires In"]
assert "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds" in (
expiry["targets"][0]["expr"]
)
assert expiry["fieldConfig"]["defaults"]["mappings"][0]["options"]["-1"][
"text"
] == "unavailable"
rules = next(
yaml.safe_load_all(
(
ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
).read_text()
)
)
quota_rules = yaml.safe_load(rules["data"]["hermes-ai-quota.yaml"])
alerts = {
rule["alert"]: rule
for group in quota_rules["groups"]
for rule in group["rules"]
}
assert "HermesClaudeQuotaCredentialExpiring" in alerts
assert "HermesClaudeFableQuotaUnavailable" in alerts
assert all(
rule["labels"]["service"] == "hermes-ai-quota"
for rule in alerts.values()
)
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
"""The first three KPI bands must compare providers without layout gaps."""
mod = load_module()

14
testing/tests/test_hermes_ai_usage_claude_scoped.py
View File

@ -144,6 +144,20 @@ def test_scoped_query_uses_current_private_access_token(tmp_path, monkeypatch):
assert requests[0].get_header("Authorization") == "Bearer scoped-access"
def test_credential_refresh_expiry_exposes_only_a_timestamp(tmp_path, monkeypatch):
mod = load_module()
credentials = tmp_path / ".credentials.json"
write_credentials(credentials, refreshTokenExpiresAt=2_000_000)
monkeypatch.setattr(mod, "CREDENTIALS_FILE", credentials)
assert mod.credential_refresh_expiry_timestamp() == 2_000
write_credentials(credentials, refreshTokenExpiresAt=True)
assert mod.credential_refresh_expiry_timestamp() is None
credentials.unlink()
assert mod.credential_refresh_expiry_timestamp() is None
def test_expired_scoped_token_refreshes_and_persists_rotation(
tmp_path, monkeypatch
):

15
testing/tests/test_hermes_ai_usage_exporter.py
View File

@ -139,8 +139,15 @@ def test_claude_payload_uses_only_supported_quota_fields():
assert secret not in repr(samples)
def test_render_reports_failure_and_freshness_without_logging_credentials():
def test_render_reports_failure_and_freshness_without_logging_credentials(
monkeypatch,
):
mod = load_module()
monkeypatch.setattr(
mod.claude_query,
"credential_refresh_expiry_timestamp",
lambda: 1_800_000_000,
)
collector = mod.Collector()
state = collector._providers["openai"]
state.samples = [
@ -164,6 +171,10 @@ def test_render_reports_failure_and_freshness_without_logging_credentials():
'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42'
in rendered
)
assert (
'atlas_ai_quota_credential_refresh_expiry_timestamp_seconds{provider="anthropic"} 1800000000'
in rendered
)
assert "accessToken" not in rendered
assert "refreshToken" not in rendered
@ -274,7 +285,7 @@ def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module():
environment = {item["name"]: item["value"] for item in exporter["env"]}
assert annotations["ai.bstein.dev/config-rev"] == (
"20260824-claude-fable-quota"
"20260825-claude-quota-expiry"
)
assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45"
assert environment["ATLAS_AI_AUTHENTICATION_GRACE_SECONDS"] == "1200"