titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

monitoring(ai): add provider quota operations dashboard

Browse Source
This commit is contained in:
jenkins 2026-08-16 05:13:20 -03:00
parent 012e5fc2ba
commit 612cefad23
11 changed files with 4262 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

319
scripts/render/dashboards_render_atlas.py
View File

@ -1884,6 +1884,7 @@ DASHBOARD_LINK_TITLES = {
"atlas-mail": "Open Atlas Mail",
"atlas-jobs": "Atlas Testing",
"atlas-testing": "Atlas Testing",
"atlas-ai": "Open Atlas AI Operations",
"atlas-power": "Open Atlas Power",
"atlas-gitops": "Open Atlas GitOps",
"atlas-gpu": "Open Atlas GPU",
@ -5153,6 +5154,320 @@ def build_testing_dashboard():
return dashboard
def build_ai_dashboard():
"""Build the internal AI quota, routing, and workload dashboard."""
remaining_thresholds = {
"mode": "absolute",
"steps": [
{"color": "gray", "value": None},
{"color": "red", "value": 0},
{"color": "orange", "value": 15},
{"color": "yellow", "value": 30},
{"color": "green", "value": 50},
],
}
unavailable_mapping = [
{"type": "value", "options": {"-1": {"text": "unavailable", "color": "gray"}}}
]
def quota_stat(panel_id, title, expr, grid, *, unit="percent"):
thresholds = remaining_thresholds
if unit != "percent":
thresholds = {
"mode": "absolute",
"steps": [
{"color": "gray", "value": None},
{"color": "blue", "value": 0},
],
}
panel = stat_panel(
panel_id,
title,
f"({expr}) or on() vector(-1)",
grid,
unit=unit,
decimals=1,
thresholds=thresholds,
instant=True,
description="Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.",
)
panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping
return panel
openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1'
anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1'
quota = "atlas_ai_quota_remaining_percent"
reset = "atlas_ai_quota_reset_timestamp_seconds"
panels = [
quota_stat(
1,
"Codex Weekly Remaining",
f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_ok})',
{"h": 4, "w": 4, "x": 0, "y": 0},
),
quota_stat(
2,
"Codex Spark Weekly Remaining",
f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_ok})',
{"h": 4, "w": 4, "x": 4, "y": 0},
),
quota_stat(
3,
"Claude 5h Remaining",
f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})',
{"h": 4, "w": 4, "x": 8, "y": 0},
),
quota_stat(
4,
"Claude 7d Remaining",
f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})',
{"h": 4, "w": 4, "x": 12, "y": 0},
),
stat_panel(
5,
"Quota Collectors Healthy",
"sum(atlas_ai_quota_fetch_success) or on() vector(0)",
{"h": 4, "w": 4, "x": 16, "y": 0},
instant=True,
thresholds={
"mode": "absolute",
"steps": [
{"color": "red", "value": None},
{"color": "yellow", "value": 1},
{"color": "green", "value": 2},
],
},
value_suffix=" / 2",
description="Successful latest quota fetches. Providers are polled independently every five minutes.",
),
stat_panel(
6,
"Oldest Quota Sample",
"max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)",
{"h": 4, "w": 4, "x": 20, "y": 0},
unit="s",
instant=True,
thresholds={
"mode": "absolute",
"steps": [
{"color": "gray", "value": None},
{"color": "green", "value": 0},
{"color": "yellow", "value": 600},
{"color": "red", "value": 1200},
],
},
description="Age of the stalest successful provider quota snapshot.",
),
quota_stat(
7,
"Codex Weekly Reset In",
f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_ok})',
{"h": 4, "w": 4, "x": 0, "y": 4},
unit="s",
),
quota_stat(
8,
"Claude 5h Reset In",
f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})',
{"h": 4, "w": 4, "x": 4, "y": 4},
unit="s",
),
quota_stat(
9,
"Claude 7d Reset In",
f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})',
{"h": 4, "w": 4, "x": 8, "y": 4},
unit="s",
),
quota_stat(
10,
"Codex Tokens (Latest Day)",
f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_ok})',
{"h": 4, "w": 4, "x": 12, "y": 4},
unit="short",
),
quota_stat(
11,
"Codex Tokens (7d)",
f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_ok})',
{"h": 4, "w": 4, "x": 16, "y": 4},
unit="short",
),
stat_panel(
12,
"Switchyard Requests",
"sum(increase(switchyard_requests_total[$__range])) or on() vector(0)",
{"h": 4, "w": 4, "x": 20, "y": 4},
unit="short",
decimals=0,
instant=True,
description="Hosted model requests observed by Switchyard in the selected dashboard range.",
),
timeseries_panel(
13,
"Model Selection Rate",
"sum by (selected_model) (rate(switchyard_decisions_total[5m]))",
{"h": 8, "w": 12, "x": 0, "y": 8},
unit="reqps",
legend="{{selected_model}}",
legend_placement="right",
description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.",
),
bargauge_panel(
14,
"Provider Selections (Range)",
'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))',
{"h": 8, "w": 12, "x": 12, "y": 8},
unit="short",
legend="{{provider}}",
instant=True,
include_color=False,
description="Switchyard selections grouped by provider over the selected dashboard range.",
),
timeseries_panel(
15,
"Token Throughput",
None,
{"h": 8, "w": 12, "x": 0, "y": 16},
unit="tps",
targets=[
{"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"},
{"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"},
{"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"},
{"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"},
{"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"},
],
description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.",
),
timeseries_panel(
16,
"Model Call p95 Latency",
"histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))",
{"h": 8, "w": 12, "x": 12, "y": 16},
unit="ms",
legend="{{model}}",
legend_placement="right",
description="95th percentile upstream latency for each selected model route.",
),
stat_panel(
17,
"Prompt Cache Share",
"100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)",
{"h": 4, "w": 6, "x": 0, "y": 24},
unit="percent",
decimals=1,
description="Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.",
),
stat_panel(
18,
"Client Success Rate",
'100 * sum(increase(switchyard_client_responses_total{outcome="success"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)',
{"h": 4, "w": 6, "x": 6, "y": 24},
unit="percent",
decimals=1,
thresholds={
"mode": "absolute",
"steps": [
{"color": "red", "value": None},
{"color": "yellow", "value": 95},
{"color": "green", "value": 99},
],
},
description="Successful client-facing Switchyard responses in the selected range.",
),
stat_panel(
19,
"Classifier Fail-Open (Range)",
"sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)",
{"h": 4, "w": 6, "x": 12, "y": 24},
decimals=0,
thresholds={
"mode": "absolute",
"steps": [
{"color": "green", "value": None},
{"color": "yellow", "value": 1},
{"color": "red", "value": 5},
],
},
description="Local classifier failures that safely fell back to the conservative hosted route.",
),
stat_panel(
20,
"Upstream Errors (Range)",
"sum(increase(switchyard_errors_total[$__range])) or on() vector(0)",
{"h": 4, "w": 6, "x": 18, "y": 24},
decimals=0,
thresholds={
"mode": "absolute",
"steps": [
{"color": "green", "value": None},
{"color": "yellow", "value": 1},
{"color": "red", "value": 5},
],
},
description="Hosted model attempts that returned errors in the selected range.",
),
timeseries_panel(
21,
"Local Classifier Calls",
'sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~"qwen.*"}[5m]))',
{"h": 8, "w": 8, "x": 0, "y": 28},
unit="reqps",
legend="{{outcome}}",
description="Local Qwen routing-classifier activity, split by successful and failed calls.",
),
timeseries_panel(
22,
"Routing Overhead p95",
"histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))",
{"h": 8, "w": 8, "x": 8, "y": 28},
unit="ms",
legend="{{algorithm}}",
description="95th percentile time Switchyard spends selecting a model before the upstream call.",
),
bargauge_panel(
23,
"Traffic Lanes (Range)",
'sum by (lane) (label_replace(increase(switchyard_requests_total{model=~"(route|worker)/.*"}[$__range]), "lane", "$1", "model", "^(route|worker)/.*"))',
{"h": 8, "w": 8, "x": 16, "y": 28},
unit="short",
legend="{{lane}}",
instant=True,
include_color=False,
description="Request volume split between interactive route traffic and durable worker traffic.",
),
text_panel(
24,
"Cost Semantics",
"Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage.",
{"h": 8, "w": 12, "x": 0, "y": 36},
),
timeseries_panel(
25,
"Hermes Workload CPU (Attribution Proxy)",
'sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace="hermes",pod=~"hermes-(agent|chat-tenant|switchyard|model-gate).*",container!="",image!=""}[5m]))',
{"h": 8, "w": 12, "x": 12, "y": 36},
unit="cores",
legend="{{pod}} · {{container}}",
legend_placement="right",
description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.",
),
]
return {
"uid": "atlas-ai",
"title": "Atlas AI Operations",
"folderUid": PRIVATE_FOLDER,
"editable": True,
"panels": panels,
"time": {"from": "now-24h", "to": "now"},
"refresh": "1m",
"annotations": {"list": []},
"schemaVersion": 39,
"style": "dark",
"tags": ["atlas", "ai", "hermes", "switchyard"],
}
def build_gitops_dashboard():
gitops_success_thresholds = {
"mode": "absolute",
@ -5677,6 +5992,10 @@ DASHBOARDS = {
"builder": build_testing_dashboard,
"configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-testing.yaml",
},
"atlas-ai": {
"builder": build_ai_dashboard,
"configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-ai.yaml",
},
"atlas-gitops": {
"builder": build_gitops_dashboard,
"configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-gitops.yaml",

47
services/hermes/agent-deployment.yaml
View File

@ -25,7 +25,10 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260816-kanban-recovery-v7"
ai.bstein.dev/config-rev: "20260816-ai-usage-v1"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-agent
vault.hashicorp.com/agent-inject-secret-agent-api-key: kv/data/atlas/hermes/agent-tokens
@ -942,6 +945,48 @@ spec:
resources:
requests: {cpu: 10m, memory: 32Mi}
limits: {cpu: 100m, memory: 128Mi}
- name: ai-usage-exporter
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/ai_usage_exporter.py
ports:
- {name: ai-metrics, containerPort: 9010, protocol: TCP}
env:
- {name: HOME, value: /tmp}
- {name: CODEX_HOME, value: /runtime-access/codex}
- {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex}
- {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json}
- {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"}
- {name: ATLAS_AI_USAGE_PORT, value: "9010"}
readinessProbe:
httpGet: {path: /healthz, port: ai-metrics}
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet: {path: /healthz, port: ai-metrics}
initialDelaySeconds: 30
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: home, mountPath: /opt/data/tools, subPath: tools, readOnly: true}
- {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude}
- {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 10m, memory: 32Mi}
limits: {cpu: 250m, memory: 192Mi}
- name: image-broker
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent

1
services/hermes/kustomization.yaml
View File

@ -58,6 +58,7 @@ configMapGenerator:
- name: hermes-coordinator
namespace: hermes
files:
- ai_usage_exporter.py=scripts/ai_usage_exporter.py
- claude=scripts/claude
- claude_command_policy.py=scripts/claude_command_policy.py
- cli_lane_runner.py=scripts/cli_lane_runner.py

9
services/hermes/networkpolicy.yaml
View File

@ -110,6 +110,15 @@ spec:
ports:
- {protocol: TCP, port: 9003}
- {protocol: TCP, port: 9006}
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
podSelector:
matchLabels:
app: server
ports:
- {protocol: TCP, port: 9010}
# agent.hermes.bstein.dev is an owner-only engineering workstation. The
# browser boundary remains OAuth-protected, while its workers need to reach
# every cluster namespace, Atlas LAN service, and hosted provider endpoint.

421
services/hermes/scripts/ai_usage_exporter.py Normal file
View File

@ -0,0 +1,421 @@
#!/usr/bin/env python3
"""Export first-party coding CLI quotas without exposing account credentials."""
from __future__ import annotations
import json
import os
import selectors
import subprocess
import threading
import time
from dataclasses import dataclass, field
from datetime import UTC, date, datetime, timedelta
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from urllib.request import Request, urlopen
CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
CLAUDE_CREDENTIALS = Path(
os.environ.get(
"ATLAS_AI_CLAUDE_CREDENTIALS",
"/runtime-access/claude/.credentials.json",
)
)
CLAUDE_USAGE_URL = os.environ.get(
"ATLAS_AI_CLAUDE_USAGE_URL",
"https://api.anthropic.com/api/oauth/usage",
)
CLAUDE_WINDOWS = (
"five_hour",
"seven_day",
"seven_day_opus",
"seven_day_sonnet",
)
METRIC_HELP = {
"atlas_ai_account_tokens": "First-party account token usage for a fixed period.",
"atlas_ai_account_usage_summary": "First-party account usage summary values.",
"atlas_ai_extra_usage_enabled": "Whether metered extra usage is enabled for the account.",
"atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.",
"atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.",
"atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.",
"atlas_ai_quota_last_success_timestamp_seconds": "Unix timestamp of the latest successful quota fetch.",
"atlas_ai_quota_remaining_percent": "Remaining percentage in a first-party coding CLI quota window.",
"atlas_ai_quota_reset_timestamp_seconds": "Unix timestamp when a coding CLI quota window resets.",
"atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.",
"atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.",
}
@dataclass(frozen=True)
class Sample:
"""One Prometheus gauge sample."""
name: str
labels: dict[str, str]
value: float
@dataclass
class ProviderState:
"""Latest safe samples and fetch health for one provider."""
samples: list[Sample] = field(default_factory=list)
fetch_success: bool = False
last_attempt: float = 0
last_success: float = 0
duration: float = 0
def _number(value: Any) -> float | None:
"""Return a finite numeric value while rejecting booleans and nulls."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
converted = float(value)
return converted if converted == converted and abs(converted) != float("inf") else None
def _timestamp(value: Any) -> float | None:
"""Parse either a Unix timestamp or an ISO-8601 timestamp."""
number = _number(value)
if number is not None:
return number
if not isinstance(value, str) or not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
except ValueError:
return None
def _window_name(window: dict[str, Any], fallback: str) -> str:
"""Give common rolling windows stable, human-readable labels."""
minutes = _number(window.get("windowDurationMins"))
known = {300: "five_hour", 1440: "one_day", 10080: "seven_day"}
if minutes is not None and int(minutes) in known:
return known[int(minutes)]
return fallback
def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str:
"""Return a stable low-cardinality name for a Codex quota bucket."""
if limit_id == "codex":
return "codex"
name = snapshot.get("limitName")
if isinstance(name, str) and name:
normalized = "".join(char.lower() if char.isalnum() else "-" for char in name)
return "-".join(filter(None, normalized.split("-")))[:64]
return "additional"
def parse_codex_payloads(
rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None
) -> list[Sample]:
"""Convert structured Codex app-server responses into bounded metrics."""
samples: list[Sample] = []
limits = rate_response.get("rateLimitsByLimitId")
if not isinstance(limits, dict) or not limits:
limits = {"codex": rate_response.get("rateLimits")}
for limit_id, snapshot in limits.items():
if not isinstance(limit_id, str) or not isinstance(snapshot, dict):
continue
limit_name = _codex_limit_name(limit_id, snapshot)
for fallback, raw_window in (
("primary", snapshot.get("primary")),
("secondary", snapshot.get("secondary")),
):
if not isinstance(raw_window, dict):
continue
used = _number(raw_window.get("usedPercent"))
if used is None:
continue
labels = {
"provider": "openai",
"limit": limit_name,
"window": _window_name(raw_window, fallback),
}
samples.extend(
(
Sample("atlas_ai_quota_used_percent", labels, used),
Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)),
)
)
reset = _timestamp(raw_window.get("resetsAt"))
duration = _number(raw_window.get("windowDurationMins"))
if reset is not None:
samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
if duration is not None:
samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60))
summary = usage_response.get("summary")
if isinstance(summary, dict):
for source, metric in (
("lifetimeTokens", "lifetime_tokens"),
("peakDailyTokens", "peak_daily_tokens"),
("currentStreakDays", "current_streak_days"),
("longestStreakDays", "longest_streak_days"),
("longestRunningTurnSec", "longest_running_turn_seconds"),
):
value = _number(summary.get(source))
if value is not None:
samples.append(
Sample(
"atlas_ai_account_usage_summary",
{"provider": "openai", "metric": metric},
value,
)
)
current_day = today or datetime.now(UTC).date()
daily = usage_response.get("dailyUsageBuckets")
parsed_daily: list[tuple[date, float]] = []
if isinstance(daily, list):
for bucket in daily:
if not isinstance(bucket, dict):
continue
value = _number(bucket.get("tokens"))
try:
start = date.fromisoformat(str(bucket.get("startDate")))
except ValueError:
continue
if value is not None:
parsed_daily.append((start, value))
# The account endpoint publishes completed daily buckets and may not include
# the current UTC day. Anchor fixed periods to the latest reported day so a
# delayed bucket is not mislabeled as zero usage.
anchor_day = max((start for start, _ in parsed_daily), default=current_day)
for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)):
earliest = anchor_day - timedelta(days=days - 1)
value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day)
samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value))
return samples
def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
"""Convert Claude's first-party OAuth usage document into bounded metrics."""
samples: list[Sample] = []
for window_name in CLAUDE_WINDOWS:
window = payload.get(window_name)
if not isinstance(window, dict):
continue
used = _number(window.get("utilization"))
if used is None:
continue
labels = {"provider": "anthropic", "limit": "claude", "window": window_name}
samples.extend(
(
Sample("atlas_ai_quota_used_percent", labels, used),
Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)),
)
)
reset = _timestamp(window.get("resets_at"))
if reset is not None:
samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
extra = payload.get("extra_usage")
if isinstance(extra, dict):
enabled = extra.get("is_enabled")
if isinstance(enabled, bool):
samples.append(
Sample(
"atlas_ai_extra_usage_enabled",
{"provider": "anthropic"},
float(enabled),
)
)
return samples
def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
"""Read Codex account quota and usage through its structured app-server protocol."""
process = subprocess.Popen(
[CODEX_BIN, "app-server", "--stdio"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
bufsize=1,
env={**os.environ, "CODEX_HOME": CODEX_HOME},
)
requests = (
{
"id": 1,
"method": "initialize",
"params": {
"clientInfo": {
"name": "atlas-ai-usage-exporter",
"title": "Atlas AI Usage Exporter",
"version": "1.0.0",
},
"capabilities": {"experimentalApi": True},
},
},
{"method": "initialized", "params": {}},
{"id": 2, "method": "account/rateLimits/read", "params": None},
{"id": 3, "method": "account/usage/read", "params": None},
)
try:
if process.stdin is None or process.stdout is None:
raise RuntimeError("Codex app-server pipes are unavailable")
for message in requests:
process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n")
process.stdin.flush()
selector = selectors.DefaultSelector()
selector.register(process.stdout, selectors.EVENT_READ)
responses: dict[int, dict[str, Any]] = {}
deadline = time.monotonic() + timeout
while len(responses) < 3 and time.monotonic() < deadline:
for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))):
line = key.fileobj.readline()
if not line:
continue
message = json.loads(line)
if message.get("id") in (1, 2, 3):
responses[int(message["id"])] = message
for response_id in (1, 2, 3):
response = responses.get(response_id)
if not response or "error" in response or not isinstance(response.get("result"), dict):
raise RuntimeError(f"Codex app-server response {response_id} failed")
return responses[2]["result"], responses[3]["result"]
finally:
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
def query_claude() -> dict[str, Any]:
"""Read Claude account quota with the runtime OAuth token held only in memory."""
document = json.loads(CLAUDE_CREDENTIALS.read_text(encoding="utf-8"))
token = document.get("claudeAiOauth", {}).get("accessToken")
if not isinstance(token, str) or not token:
raise RuntimeError("Claude runtime credentials are incomplete")
request = Request(
CLAUDE_USAGE_URL,
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/json",
"anthropic-version": "2023-06-01",
"User-Agent": "atlas-ai-usage-exporter/1.0",
},
)
with urlopen(request, timeout=15) as response:
payload = json.loads(response.read(1 << 20))
if not isinstance(payload, dict):
raise RuntimeError("Claude usage response is not an object")
return payload
def _escape_label(value: str) -> str:
"""Escape one Prometheus label value."""
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
class Collector:
"""Poll providers independently and expose a thread-safe metrics snapshot."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._providers = {name: ProviderState() for name in ("openai", "anthropic")}
def refresh_provider(self, provider: str) -> None:
"""Refresh one provider while retaining the last good values on failure."""
started = time.time()
monotonic_started = time.monotonic()
try:
if provider == "openai":
samples = parse_codex_payloads(*query_codex())
elif provider == "anthropic":
samples = parse_claude_payload(query_claude())
else:
raise ValueError("unknown provider")
success = True
except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error:
print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True)
samples = []
success = False
with self._lock:
state = self._providers[provider]
state.last_attempt = started
state.duration = time.monotonic() - monotonic_started
state.fetch_success = success
if success:
state.samples = samples
state.last_success = time.time()
def render(self) -> bytes:
"""Render the current provider states in Prometheus text format."""
with self._lock:
states = {
provider: ProviderState(**vars(state))
for provider, state in self._providers.items()
}
samples: list[Sample] = []
for provider, state in states.items():
labels = {"provider": provider}
samples.extend(state.samples)
samples.extend(
(
Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)),
Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt),
Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success),
Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration),
)
)
lines: list[str] = []
for name in sorted({sample.name for sample in samples}):
lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} gauge"))
for sample in sorted(
(item for item in samples if item.name == name),
key=lambda item: sorted(item.labels.items()),
):
labels = ",".join(
f'{key}="{_escape_label(value)}"'
for key, value in sorted(sample.labels.items())
)
lines.append(f"{name}{{{labels}}} {sample.value:.12g}")
return ("\n".join(lines) + "\n").encode("utf-8")
def main() -> int:
"""Poll quota APIs and serve only sanitized metrics and health endpoints."""
collector = Collector()
interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300")))
def polling_loop() -> None:
while True:
for provider in ("openai", "anthropic"):
collector.refresh_provider(provider)
time.sleep(interval)
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
if self.path == "/metrics":
payload = collector.render()
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
elif self.path == "/healthz":
self.send_response(200)
self.end_headers()
else:
self.send_error(404)
def log_message(self, _format: str, *_args: object) -> None:
return
threading.Thread(target=polling_loop, daemon=True).start()
port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010"))
server = ThreadingHTTPServer(("0.0.0.0", port), Handler)
server.serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())

1603
services/monitoring/dashboards/atlas-ai.json Normal file
View File

File diff suppressed because it is too large Load Diff

1612
services/monitoring/grafana-dashboard-ai.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

10
services/monitoring/helmrelease.yaml
View File

@ -622,6 +622,15 @@ spec:
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards/testing-public
- name: ai
orgId: 1
folder: Atlas Internal
type: file
disableDeletion: false
editable: true
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards/ai
- name: power
orgId: 1
folder: Atlas Internal
@ -651,6 +660,7 @@ spec:
mail: grafana-dashboard-mail
testing: grafana-dashboard-testing
testing-public: grafana-dashboard-testing
ai: grafana-dashboard-ai
power: grafana-dashboard-power
cassandra: grafana-dashboard-cassandra
extraConfigmapMounts:

1
services/monitoring/kustomization.yaml
View File

@ -18,6 +18,7 @@ resources:
- grafana-dashboard-power.yaml
- grafana-dashboard-mail.yaml
- grafana-dashboard-testing.yaml
- grafana-dashboard-ai.yaml
- vmalert-atlas-availability.yaml
- availability-backfill-v4-job.yaml
- availability-daily-backfill-v4-job.yaml

65
testing/tests/test_atlas_ai_dashboard.py Normal file
View File

@ -0,0 +1,65 @@
"""Contract checks for the internal Atlas AI operations dashboard."""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py"
def load_module():
"""Load the dashboard generator without invoking its CLI."""
spec = importlib.util.spec_from_file_location("dashboards_render_atlas_ai", SCRIPT)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
mod = load_module()
dashboard = mod.build_ai_dashboard()
panels = {panel["title"]: panel for panel in dashboard["panels"]}
expressions = "\n".join(
target.get("expr", "")
for panel in dashboard["panels"]
for target in panel.get("targets", [])
)
assert dashboard["uid"] == "atlas-ai"
assert dashboard["folderUid"] == mod.PRIVATE_FOLDER
assert dashboard["refresh"] == "1m"
assert "Codex Weekly Remaining" in panels
assert "Claude 5h Remaining" in panels
assert "Claude 7d Remaining" in panels
assert "Provider Selections (Range)" in panels
assert "Local Classifier Calls" in panels
assert "Hermes Workload CPU (Attribution Proxy)" in panels
assert "atlas_ai_quota_remaining_percent" in expressions
assert "atlas_ai_quota_reset_timestamp_seconds" in expressions
assert "switchyard_decisions_total" in expressions
assert "switchyard_cached_tokens_total" in expressions
assert "switchyard_model_call_latency_ms_bucket" in expressions
assert "/status" not in expressions
assert all(
panel.get("description") or panel["type"] == "text"
for panel in dashboard["panels"]
)
def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution():
mod = load_module()
dashboard = mod.build_ai_dashboard()
panels = {panel["title"]: panel for panel in dashboard["panels"]}
cost_text = panels["Cost Semantics"]["options"]["content"]
proxy_description = panels["Hermes Workload CPU (Attribution Proxy)"]["description"]
assert "does not invent API costs" in cost_text
assert "subscription" in cost_text
assert "proxy" in proxy_description
assert "not tenant-slot token labels" in proxy_description

175
testing/tests/test_hermes_ai_usage_exporter.py Normal file
View File

@ -0,0 +1,175 @@
"""Tests for the credential-safe Atlas AI usage exporter."""
from __future__ import annotations
import importlib.util
import json
import sys
from datetime import date
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py"
def load_module():
"""Load the standalone exporter script as a test module."""
spec = importlib.util.spec_from_file_location("ai_usage_exporter", SCRIPT)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_codex_payloads_include_real_windows_and_bounded_token_periods():
mod = load_module()
rate = {
"rateLimitsByLimitId": {
"codex": {
"primary": {
"usedPercent": 41,
"windowDurationMins": 10080,
"resetsAt": 1_800_000_000,
}
},
"opaque_backend_id": {
"limitName": "GPT-5.3-Codex-Spark",
"primary": {
"usedPercent": 0,
"windowDurationMins": 10080,
"resetsAt": 1_800_100_000,
},
},
}
}
usage = {
"summary": {"lifetimeTokens": 1234, "peakDailyTokens": 500},
"dailyUsageBuckets": [
{"startDate": "2026-08-16", "tokens": 100},
{"startDate": "2026-08-15", "tokens": 50},
{"startDate": "2026-07-01", "tokens": 900},
],
}
samples = mod.parse_codex_payloads(rate, usage, today=date(2026, 8, 16))
values = {(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples}
codex_labels = (
("limit", "codex"),
("provider", "openai"),
("window", "seven_day"),
)
spark_labels = (
("limit", "gpt-5-3-codex-spark"),
("provider", "openai"),
("window", "seven_day"),
)
assert values[("atlas_ai_quota_used_percent", codex_labels)] == 41
assert values[("atlas_ai_quota_remaining_percent", codex_labels)] == 59
assert values[("atlas_ai_quota_used_percent", spark_labels)] == 0
assert values[
(
"atlas_ai_account_tokens",
(("period", "latest_day"), ("provider", "openai")),
)
] == 100
assert values[
(
"atlas_ai_account_tokens",
(("period", "seven_day"), ("provider", "openai")),
)
] == 150
def test_claude_payload_uses_only_supported_quota_fields():
mod = load_module()
secret = "secret-access-token-must-not-leak"
payload = {
"five_hour": {
"utilization": 16.0,
"resets_at": "2026-08-16T08:39:59+00:00",
"unexpected_secret": secret,
},
"seven_day": {
"utilization": 26.0,
"resets_at": "2026-08-20T17:59:59+00:00",
},
"internal_bucket": {"utilization": 99, "resets_at": secret},
"extra_usage": {"is_enabled": False, "used_credits": secret},
}
samples = mod.parse_claude_payload(payload)
assert {item.labels["window"] for item in samples if "window" in item.labels} == {
"five_hour",
"seven_day",
}
assert any(
item.name == "atlas_ai_quota_remaining_percent"
and item.labels["window"] == "seven_day"
and item.value == 74
for item in samples
)
assert secret not in repr(samples)
def test_render_reports_failure_and_freshness_without_logging_credentials():
mod = load_module()
collector = mod.Collector()
state = collector._providers["openai"]
state.samples = [
mod.Sample(
"atlas_ai_quota_used_percent",
{"provider": "openai", "limit": "codex", "window": "seven_day"},
42,
)
]
state.fetch_success = False
state.last_attempt = 200
state.last_success = 100
rendered = collector.render().decode()
assert 'atlas_ai_quota_fetch_success{provider="openai"} 0' in rendered
assert 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' in rendered
assert 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered
assert "accessToken" not in rendered
assert "refreshToken" not in rendered
def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch):
mod = load_module()
request_log = tmp_path / "requests.jsonl"
mock = tmp_path / "codex-mock"
mock.write_text(
"""#!/usr/bin/env python3
import json, os, sys
messages = [json.loads(sys.stdin.readline()) for _ in range(4)]
with open(os.environ['REQUEST_LOG'], 'w') as handle:
for message in messages:
handle.write(json.dumps(message) + '\\n')
responses = {
1: {},
2: {'rateLimits': {'primary': {'usedPercent': 10}}},
3: {'summary': {}, 'dailyUsageBuckets': []},
}
for response_id, result in responses.items():
print(json.dumps({'id': response_id, 'result': result}), flush=True)
"""
)
mock.chmod(0o755)
monkeypatch.setattr(mod, "CODEX_BIN", str(mock))
monkeypatch.setenv("REQUEST_LOG", str(request_log))
rate, usage = mod.query_codex(timeout=5)
requests = [json.loads(line) for line in request_log.read_text().splitlines()]
assert rate["rateLimits"]["primary"]["usedPercent"] == 10
assert usage["summary"] == {}
assert [request["method"] for request in requests] == [
"initialize",
"initialized",
"account/rateLimits/read",
"account/usage/read",
]
assert all("/status" not in json.dumps(request) for request in requests)