From 612cefad23f5202e8472f987627c2ece065d8bb5 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 05:13:20 -0300 Subject: [PATCH] monitoring(ai): add provider quota operations dashboard --- scripts/render/dashboards_render_atlas.py | 319 ++++ services/hermes/agent-deployment.yaml | 47 +- services/hermes/kustomization.yaml | 1 + services/hermes/networkpolicy.yaml | 9 + services/hermes/scripts/ai_usage_exporter.py | 421 +++++ services/monitoring/dashboards/atlas-ai.json | 1603 ++++++++++++++++ services/monitoring/grafana-dashboard-ai.yaml | 1612 +++++++++++++++++ services/monitoring/helmrelease.yaml | 10 + services/monitoring/kustomization.yaml | 1 + testing/tests/test_atlas_ai_dashboard.py | 65 + .../tests/test_hermes_ai_usage_exporter.py | 175 ++ 11 files changed, 4262 insertions(+), 1 deletion(-) create mode 100644 services/hermes/scripts/ai_usage_exporter.py create mode 100644 services/monitoring/dashboards/atlas-ai.json create mode 100644 services/monitoring/grafana-dashboard-ai.yaml create mode 100644 testing/tests/test_atlas_ai_dashboard.py create mode 100644 testing/tests/test_hermes_ai_usage_exporter.py diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index e8c9989f..c0b462f6 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -1884,6 +1884,7 @@ DASHBOARD_LINK_TITLES = { "atlas-mail": "Open Atlas Mail", "atlas-jobs": "Atlas Testing", "atlas-testing": "Atlas Testing", + "atlas-ai": "Open Atlas AI Operations", "atlas-power": "Open Atlas Power", "atlas-gitops": "Open Atlas GitOps", "atlas-gpu": "Open Atlas GPU", @@ -5153,6 +5154,320 @@ def build_testing_dashboard(): return dashboard +def build_ai_dashboard(): + """Build the internal AI quota, routing, and workload dashboard.""" + remaining_thresholds = { + "mode": "absolute", + "steps": [ + {"color": "gray", "value": None}, + {"color": "red", "value": 0}, + {"color": "orange", "value": 15}, + {"color": "yellow", "value": 30}, + {"color": "green", "value": 50}, + ], + } + unavailable_mapping = [ + {"type": "value", "options": {"-1": {"text": "unavailable", "color": "gray"}}} + ] + + def quota_stat(panel_id, title, expr, grid, *, unit="percent"): + thresholds = remaining_thresholds + if unit != "percent": + thresholds = { + "mode": "absolute", + "steps": [ + {"color": "gray", "value": None}, + {"color": "blue", "value": 0}, + ], + } + panel = stat_panel( + panel_id, + title, + f"({expr}) or on() vector(-1)", + grid, + unit=unit, + decimals=1, + thresholds=thresholds, + instant=True, + description="Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + ) + panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping + return panel + + openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1' + anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1' + quota = "atlas_ai_quota_remaining_percent" + reset = "atlas_ai_quota_reset_timestamp_seconds" + panels = [ + quota_stat( + 1, + "Codex Weekly Remaining", + f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_ok})', + {"h": 4, "w": 4, "x": 0, "y": 0}, + ), + quota_stat( + 2, + "Codex Spark Weekly Remaining", + f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_ok})', + {"h": 4, "w": 4, "x": 4, "y": 0}, + ), + quota_stat( + 3, + "Claude 5h Remaining", + f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})', + {"h": 4, "w": 4, "x": 8, "y": 0}, + ), + quota_stat( + 4, + "Claude 7d Remaining", + f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})', + {"h": 4, "w": 4, "x": 12, "y": 0}, + ), + stat_panel( + 5, + "Quota Collectors Healthy", + "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + {"h": 4, "w": 4, "x": 16, "y": 0}, + instant=True, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "red", "value": None}, + {"color": "yellow", "value": 1}, + {"color": "green", "value": 2}, + ], + }, + value_suffix=" / 2", + description="Successful latest quota fetches. Providers are polled independently every five minutes.", + ), + stat_panel( + 6, + "Oldest Quota Sample", + "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + {"h": 4, "w": 4, "x": 20, "y": 0}, + unit="s", + instant=True, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "gray", "value": None}, + {"color": "green", "value": 0}, + {"color": "yellow", "value": 600}, + {"color": "red", "value": 1200}, + ], + }, + description="Age of the stalest successful provider quota snapshot.", + ), + quota_stat( + 7, + "Codex Weekly Reset In", + f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_ok})', + {"h": 4, "w": 4, "x": 0, "y": 4}, + unit="s", + ), + quota_stat( + 8, + "Claude 5h Reset In", + f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})', + {"h": 4, "w": 4, "x": 4, "y": 4}, + unit="s", + ), + quota_stat( + 9, + "Claude 7d Reset In", + f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})', + {"h": 4, "w": 4, "x": 8, "y": 4}, + unit="s", + ), + quota_stat( + 10, + "Codex Tokens (Latest Day)", + f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_ok})', + {"h": 4, "w": 4, "x": 12, "y": 4}, + unit="short", + ), + quota_stat( + 11, + "Codex Tokens (7d)", + f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_ok})', + {"h": 4, "w": 4, "x": 16, "y": 4}, + unit="short", + ), + stat_panel( + 12, + "Switchyard Requests", + "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)", + {"h": 4, "w": 4, "x": 20, "y": 4}, + unit="short", + decimals=0, + instant=True, + description="Hosted model requests observed by Switchyard in the selected dashboard range.", + ), + timeseries_panel( + 13, + "Model Selection Rate", + "sum by (selected_model) (rate(switchyard_decisions_total[5m]))", + {"h": 8, "w": 12, "x": 0, "y": 8}, + unit="reqps", + legend="{{selected_model}}", + legend_placement="right", + description="AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.", + ), + bargauge_panel( + 14, + "Provider Selections (Range)", + 'sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~"(route|worker)/(codex|claude|local)/.*"}[$__range]), "provider", "$2", "selected_model", "^(route|worker)/(codex|claude|local)/.*"))', + {"h": 8, "w": 12, "x": 12, "y": 8}, + unit="short", + legend="{{provider}}", + instant=True, + include_color=False, + description="Switchyard selections grouped by provider over the selected dashboard range.", + ), + timeseries_panel( + 15, + "Token Throughput", + None, + {"h": 8, "w": 12, "x": 0, "y": 16}, + unit="tps", + targets=[ + {"expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", "refId": "A", "legendFormat": "prompt"}, + {"expr": "sum(rate(switchyard_cached_tokens_total[5m]))", "refId": "B", "legendFormat": "cached"}, + {"expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", "refId": "C", "legendFormat": "cache creation"}, + {"expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", "refId": "D", "legendFormat": "reasoning"}, + {"expr": "sum(rate(switchyard_completion_tokens_total[5m]))", "refId": "E", "legendFormat": "completion"}, + ], + description="Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.", + ), + timeseries_panel( + 16, + "Model Call p95 Latency", + "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))", + {"h": 8, "w": 12, "x": 12, "y": 16}, + unit="ms", + legend="{{model}}", + legend_placement="right", + description="95th percentile upstream latency for each selected model route.", + ), + stat_panel( + 17, + "Prompt Cache Share", + "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", + {"h": 4, "w": 6, "x": 0, "y": 24}, + unit="percent", + decimals=1, + description="Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.", + ), + stat_panel( + 18, + "Client Success Rate", + '100 * sum(increase(switchyard_client_responses_total{outcome="success"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)', + {"h": 4, "w": 6, "x": 6, "y": 24}, + unit="percent", + decimals=1, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "red", "value": None}, + {"color": "yellow", "value": 95}, + {"color": "green", "value": 99}, + ], + }, + description="Successful client-facing Switchyard responses in the selected range.", + ), + stat_panel( + 19, + "Classifier Fail-Open (Range)", + "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", + {"h": 4, "w": 6, "x": 12, "y": 24}, + decimals=0, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 5}, + ], + }, + description="Local classifier failures that safely fell back to the conservative hosted route.", + ), + stat_panel( + 20, + "Upstream Errors (Range)", + "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", + {"h": 4, "w": 6, "x": 18, "y": 24}, + decimals=0, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 5}, + ], + }, + description="Hosted model attempts that returned errors in the selected range.", + ), + timeseries_panel( + 21, + "Local Classifier Calls", + 'sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~"qwen.*"}[5m]))', + {"h": 8, "w": 8, "x": 0, "y": 28}, + unit="reqps", + legend="{{outcome}}", + description="Local Qwen routing-classifier activity, split by successful and failed calls.", + ), + timeseries_panel( + 22, + "Routing Overhead p95", + "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))", + {"h": 8, "w": 8, "x": 8, "y": 28}, + unit="ms", + legend="{{algorithm}}", + description="95th percentile time Switchyard spends selecting a model before the upstream call.", + ), + bargauge_panel( + 23, + "Traffic Lanes (Range)", + 'sum by (lane) (label_replace(increase(switchyard_requests_total{model=~"(route|worker)/.*"}[$__range]), "lane", "$1", "model", "^(route|worker)/.*"))', + {"h": 8, "w": 8, "x": 16, "y": 28}, + unit="short", + legend="{{lane}}", + instant=True, + include_color=False, + description="Request volume split between interactive route traffic and durable worker traffic.", + ), + text_panel( + 24, + "Cost Semantics", + "Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage.", + {"h": 8, "w": 12, "x": 0, "y": 36}, + ), + timeseries_panel( + 25, + "Hermes Workload CPU (Attribution Proxy)", + 'sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace="hermes",pod=~"hermes-(agent|chat-tenant|switchyard|model-gate).*",container!="",image!=""}[5m]))', + {"h": 8, "w": 12, "x": 12, "y": 36}, + unit="cores", + legend="{{pod}} ยท {{container}}", + legend_placement="right", + description="Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", + ), + ] + return { + "uid": "atlas-ai", + "title": "Atlas AI Operations", + "folderUid": PRIVATE_FOLDER, + "editable": True, + "panels": panels, + "time": {"from": "now-24h", "to": "now"}, + "refresh": "1m", + "annotations": {"list": []}, + "schemaVersion": 39, + "style": "dark", + "tags": ["atlas", "ai", "hermes", "switchyard"], + } + + def build_gitops_dashboard(): gitops_success_thresholds = { "mode": "absolute", @@ -5677,6 +5992,10 @@ DASHBOARDS = { "builder": build_testing_dashboard, "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-testing.yaml", }, + "atlas-ai": { + "builder": build_ai_dashboard, + "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-ai.yaml", + }, "atlas-gitops": { "builder": build_gitops_dashboard, "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-gitops.yaml", diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 535aadf6..51e27d98 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,10 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-kanban-recovery-v7" + ai.bstein.dev/config-rev: "20260816-ai-usage-v1" + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "9010" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-agent-api-key: kv/data/atlas/hermes/agent-tokens @@ -942,6 +945,48 @@ spec: resources: requests: {cpu: 10m, memory: 32Mi} limits: {cpu: 100m, memory: 128Mi} + - name: ai-usage-exporter + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/ai_usage_exporter.py + ports: + - {name: ai-metrics, containerPort: 9010, protocol: TCP} + env: + - {name: HOME, value: /tmp} + - {name: CODEX_HOME, value: /runtime-access/codex} + - {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex} + - {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json} + - {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"} + - {name: ATLAS_AI_USAGE_PORT, value: "9010"} + readinessProbe: + httpGet: {path: /healthz, port: ai-metrics} + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: ai-metrics} + initialDelaySeconds: 30 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /opt/data/tools, subPath: tools, readOnly: true} + - {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude} + - {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 250m, memory: 192Mi} - name: image-broker image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 94d04027..8ec736e3 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -58,6 +58,7 @@ configMapGenerator: - name: hermes-coordinator namespace: hermes files: + - ai_usage_exporter.py=scripts/ai_usage_exporter.py - claude=scripts/claude - claude_command_policy.py=scripts/claude_command_policy.py - cli_lane_runner.py=scripts/cli_lane_runner.py diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index 699c97b8..0dc2c673 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -110,6 +110,15 @@ spec: ports: - {protocol: TCP, port: 9003} - {protocol: TCP, port: 9006} + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + podSelector: + matchLabels: + app: server + ports: + - {protocol: TCP, port: 9010} # agent.hermes.bstein.dev is an owner-only engineering workstation. The # browser boundary remains OAuth-protected, while its workers need to reach # every cluster namespace, Atlas LAN service, and hosted provider endpoint. diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py new file mode 100644 index 00000000..178c2c54 --- /dev/null +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +"""Export first-party coding CLI quotas without exposing account credentials.""" + +from __future__ import annotations + +import json +import os +import selectors +import subprocess +import threading +import time +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen + + +CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") +CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") +CLAUDE_CREDENTIALS = Path( + os.environ.get( + "ATLAS_AI_CLAUDE_CREDENTIALS", + "/runtime-access/claude/.credentials.json", + ) +) +CLAUDE_USAGE_URL = os.environ.get( + "ATLAS_AI_CLAUDE_USAGE_URL", + "https://api.anthropic.com/api/oauth/usage", +) +CLAUDE_WINDOWS = ( + "five_hour", + "seven_day", + "seven_day_opus", + "seven_day_sonnet", +) +METRIC_HELP = { + "atlas_ai_account_tokens": "First-party account token usage for a fixed period.", + "atlas_ai_account_usage_summary": "First-party account usage summary values.", + "atlas_ai_extra_usage_enabled": "Whether metered extra usage is enabled for the account.", + "atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.", + "atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.", + "atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.", + "atlas_ai_quota_last_success_timestamp_seconds": "Unix timestamp of the latest successful quota fetch.", + "atlas_ai_quota_remaining_percent": "Remaining percentage in a first-party coding CLI quota window.", + "atlas_ai_quota_reset_timestamp_seconds": "Unix timestamp when a coding CLI quota window resets.", + "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.", + "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.", +} + + +@dataclass(frozen=True) +class Sample: + """One Prometheus gauge sample.""" + + name: str + labels: dict[str, str] + value: float + + +@dataclass +class ProviderState: + """Latest safe samples and fetch health for one provider.""" + + samples: list[Sample] = field(default_factory=list) + fetch_success: bool = False + last_attempt: float = 0 + last_success: float = 0 + duration: float = 0 + + +def _number(value: Any) -> float | None: + """Return a finite numeric value while rejecting booleans and nulls.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + converted = float(value) + return converted if converted == converted and abs(converted) != float("inf") else None + + +def _timestamp(value: Any) -> float | None: + """Parse either a Unix timestamp or an ISO-8601 timestamp.""" + number = _number(value) + if number is not None: + return number + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except ValueError: + return None + + +def _window_name(window: dict[str, Any], fallback: str) -> str: + """Give common rolling windows stable, human-readable labels.""" + minutes = _number(window.get("windowDurationMins")) + known = {300: "five_hour", 1440: "one_day", 10080: "seven_day"} + if minutes is not None and int(minutes) in known: + return known[int(minutes)] + return fallback + + +def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str: + """Return a stable low-cardinality name for a Codex quota bucket.""" + if limit_id == "codex": + return "codex" + name = snapshot.get("limitName") + if isinstance(name, str) and name: + normalized = "".join(char.lower() if char.isalnum() else "-" for char in name) + return "-".join(filter(None, normalized.split("-")))[:64] + return "additional" + + +def parse_codex_payloads( + rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None +) -> list[Sample]: + """Convert structured Codex app-server responses into bounded metrics.""" + samples: list[Sample] = [] + limits = rate_response.get("rateLimitsByLimitId") + if not isinstance(limits, dict) or not limits: + limits = {"codex": rate_response.get("rateLimits")} + for limit_id, snapshot in limits.items(): + if not isinstance(limit_id, str) or not isinstance(snapshot, dict): + continue + limit_name = _codex_limit_name(limit_id, snapshot) + for fallback, raw_window in ( + ("primary", snapshot.get("primary")), + ("secondary", snapshot.get("secondary")), + ): + if not isinstance(raw_window, dict): + continue + used = _number(raw_window.get("usedPercent")) + if used is None: + continue + labels = { + "provider": "openai", + "limit": limit_name, + "window": _window_name(raw_window, fallback), + } + samples.extend( + ( + Sample("atlas_ai_quota_used_percent", labels, used), + Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)), + ) + ) + reset = _timestamp(raw_window.get("resetsAt")) + duration = _number(raw_window.get("windowDurationMins")) + if reset is not None: + samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + if duration is not None: + samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60)) + + summary = usage_response.get("summary") + if isinstance(summary, dict): + for source, metric in ( + ("lifetimeTokens", "lifetime_tokens"), + ("peakDailyTokens", "peak_daily_tokens"), + ("currentStreakDays", "current_streak_days"), + ("longestStreakDays", "longest_streak_days"), + ("longestRunningTurnSec", "longest_running_turn_seconds"), + ): + value = _number(summary.get(source)) + if value is not None: + samples.append( + Sample( + "atlas_ai_account_usage_summary", + {"provider": "openai", "metric": metric}, + value, + ) + ) + + current_day = today or datetime.now(UTC).date() + daily = usage_response.get("dailyUsageBuckets") + parsed_daily: list[tuple[date, float]] = [] + if isinstance(daily, list): + for bucket in daily: + if not isinstance(bucket, dict): + continue + value = _number(bucket.get("tokens")) + try: + start = date.fromisoformat(str(bucket.get("startDate"))) + except ValueError: + continue + if value is not None: + parsed_daily.append((start, value)) + # The account endpoint publishes completed daily buckets and may not include + # the current UTC day. Anchor fixed periods to the latest reported day so a + # delayed bucket is not mislabeled as zero usage. + anchor_day = max((start for start, _ in parsed_daily), default=current_day) + for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)): + earliest = anchor_day - timedelta(days=days - 1) + value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day) + samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value)) + return samples + + +def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: + """Convert Claude's first-party OAuth usage document into bounded metrics.""" + samples: list[Sample] = [] + for window_name in CLAUDE_WINDOWS: + window = payload.get(window_name) + if not isinstance(window, dict): + continue + used = _number(window.get("utilization")) + if used is None: + continue + labels = {"provider": "anthropic", "limit": "claude", "window": window_name} + samples.extend( + ( + Sample("atlas_ai_quota_used_percent", labels, used), + Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)), + ) + ) + reset = _timestamp(window.get("resets_at")) + if reset is not None: + samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + extra = payload.get("extra_usage") + if isinstance(extra, dict): + enabled = extra.get("is_enabled") + if isinstance(enabled, bool): + samples.append( + Sample( + "atlas_ai_extra_usage_enabled", + {"provider": "anthropic"}, + float(enabled), + ) + ) + return samples + + +def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: + """Read Codex account quota and usage through its structured app-server protocol.""" + process = subprocess.Popen( + [CODEX_BIN, "app-server", "--stdio"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + env={**os.environ, "CODEX_HOME": CODEX_HOME}, + ) + requests = ( + { + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "atlas-ai-usage-exporter", + "title": "Atlas AI Usage Exporter", + "version": "1.0.0", + }, + "capabilities": {"experimentalApi": True}, + }, + }, + {"method": "initialized", "params": {}}, + {"id": 2, "method": "account/rateLimits/read", "params": None}, + {"id": 3, "method": "account/usage/read", "params": None}, + ) + try: + if process.stdin is None or process.stdout is None: + raise RuntimeError("Codex app-server pipes are unavailable") + for message in requests: + process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + process.stdin.flush() + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + responses: dict[int, dict[str, Any]] = {} + deadline = time.monotonic() + timeout + while len(responses) < 3 and time.monotonic() < deadline: + for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))): + line = key.fileobj.readline() + if not line: + continue + message = json.loads(line) + if message.get("id") in (1, 2, 3): + responses[int(message["id"])] = message + for response_id in (1, 2, 3): + response = responses.get(response_id) + if not response or "error" in response or not isinstance(response.get("result"), dict): + raise RuntimeError(f"Codex app-server response {response_id} failed") + return responses[2]["result"], responses[3]["result"] + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def query_claude() -> dict[str, Any]: + """Read Claude account quota with the runtime OAuth token held only in memory.""" + document = json.loads(CLAUDE_CREDENTIALS.read_text(encoding="utf-8")) + token = document.get("claudeAiOauth", {}).get("accessToken") + if not isinstance(token, str) or not token: + raise RuntimeError("Claude runtime credentials are incomplete") + request = Request( + CLAUDE_USAGE_URL, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "anthropic-version": "2023-06-01", + "User-Agent": "atlas-ai-usage-exporter/1.0", + }, + ) + with urlopen(request, timeout=15) as response: + payload = json.loads(response.read(1 << 20)) + if not isinstance(payload, dict): + raise RuntimeError("Claude usage response is not an object") + return payload + + +def _escape_label(value: str) -> str: + """Escape one Prometheus label value.""" + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +class Collector: + """Poll providers independently and expose a thread-safe metrics snapshot.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._providers = {name: ProviderState() for name in ("openai", "anthropic")} + + def refresh_provider(self, provider: str) -> None: + """Refresh one provider while retaining the last good values on failure.""" + started = time.time() + monotonic_started = time.monotonic() + try: + if provider == "openai": + samples = parse_codex_payloads(*query_codex()) + elif provider == "anthropic": + samples = parse_claude_payload(query_claude()) + else: + raise ValueError("unknown provider") + success = True + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True) + samples = [] + success = False + with self._lock: + state = self._providers[provider] + state.last_attempt = started + state.duration = time.monotonic() - monotonic_started + state.fetch_success = success + if success: + state.samples = samples + state.last_success = time.time() + + def render(self) -> bytes: + """Render the current provider states in Prometheus text format.""" + with self._lock: + states = { + provider: ProviderState(**vars(state)) + for provider, state in self._providers.items() + } + samples: list[Sample] = [] + for provider, state in states.items(): + labels = {"provider": provider} + samples.extend(state.samples) + samples.extend( + ( + Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)), + Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt), + Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success), + Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration), + ) + ) + lines: list[str] = [] + for name in sorted({sample.name for sample in samples}): + lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} gauge")) + for sample in sorted( + (item for item in samples if item.name == name), + key=lambda item: sorted(item.labels.items()), + ): + labels = ",".join( + f'{key}="{_escape_label(value)}"' + for key, value in sorted(sample.labels.items()) + ) + lines.append(f"{name}{{{labels}}} {sample.value:.12g}") + return ("\n".join(lines) + "\n").encode("utf-8") + + +def main() -> int: + """Poll quota APIs and serve only sanitized metrics and health endpoints.""" + collector = Collector() + interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300"))) + + def polling_loop() -> None: + while True: + for provider in ("openai", "anthropic"): + collector.refresh_provider(provider) + time.sleep(interval) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + if self.path == "/metrics": + payload = collector.render() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + elif self.path == "/healthz": + self.send_response(200) + self.end_headers() + else: + self.send_error(404) + + def log_message(self, _format: str, *_args: object) -> None: + return + + threading.Thread(target=polling_loop, daemon=True).start() + port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010")) + server = ThreadingHTTPServer(("0.0.0.0", port), Handler) + server.serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json new file mode 100644 index 00000000..60b6dc9c --- /dev/null +++ b/services/monitoring/dashboards/atlas-ai.json @@ -0,0 +1,1603 @@ +{ + "uid": "atlas-ai", + "title": "Atlas AI Operations", + "folderUid": "atlas-internal", + "editable": true, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Codex Weekly Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 2, + "type": "stat", + "title": "Codex Spark Weekly Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 3, + "type": "stat", + "title": "Claude 5h Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 4, + "type": "stat", + "title": "Claude 7d Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 5, + "type": "stat", + "title": "Quota Collectors Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 2 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto", + "valueSuffix": " / 2" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful latest quota fetches. Providers are polled independently every five minutes." + }, + { + "id": 6, + "type": "stat", + "title": "Oldest Quota Sample", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-green", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 600 + }, + { + "color": "dark-red", + "value": 1200 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Age of the stalest successful provider quota snapshot." + }, + { + "id": 7, + "type": "stat", + "title": "Codex Weekly Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 8, + "type": "stat", + "title": "Claude 5h Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 9, + "type": "stat", + "title": "Claude 7d Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 10, + "type": "stat", + "title": "Codex Tokens (Latest Day)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 11, + "type": "stat", + "title": "Codex Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 4 + }, + "targets": [ + { + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 12, + "type": "stat", + "title": "Switchyard Requests", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model requests observed by Switchyard in the selected dashboard range." + }, + { + "id": 13, + "type": "timeseries", + "title": "Model Selection Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "expr": "sum by (selected_model) (rate(switchyard_decisions_total[5m]))", + "refId": "A", + "legendFormat": "{{selected_model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort." + }, + { + "id": 14, + "type": "bargauge", + "title": "Provider Selections (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "sort_desc(sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~\"(route|worker)/(codex|claude|local)/.*\"}[$__range]), \"provider\", \"$2\", \"selected_model\", \"^(route|worker)/(codex|claude|local)/.*\")))", + "refId": "A", + "legendFormat": "{{provider}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 50 + }, + { + "color": "dark-orange", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Switchyard selections grouped by provider over the selected dashboard range.", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Token Throughput", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", + "refId": "A", + "legendFormat": "prompt" + }, + { + "expr": "sum(rate(switchyard_cached_tokens_total[5m]))", + "refId": "B", + "legendFormat": "cached" + }, + { + "expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", + "refId": "C", + "legendFormat": "cache creation" + }, + { + "expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", + "refId": "D", + "legendFormat": "reasoning" + }, + { + "expr": "sum(rate(switchyard_completion_tokens_total[5m]))", + "refId": "E", + "legendFormat": "completion" + } + ], + "fieldConfig": { + "defaults": { + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls." + }, + { + "id": 16, + "type": "timeseries", + "title": "Model Call p95 Latency", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))", + "refId": "A", + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "95th percentile upstream latency for each selected model route." + }, + { + "id": 17, + "type": "stat", + "title": "Prompt Cache Share", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 24 + }, + "targets": [ + { + "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." + }, + { + "id": 18, + "type": "stat", + "title": "Client Success Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 24 + }, + "targets": [ + { + "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 99 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful client-facing Switchyard responses in the selected range." + }, + { + "id": 19, + "type": "stat", + "title": "Classifier Fail-Open (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Local classifier failures that safely fell back to the conservative hosted route." + }, + { + "id": 20, + "type": "stat", + "title": "Upstream Errors (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model attempts that returned errors in the selected range." + }, + { + "id": 21, + "type": "timeseries", + "title": "Local Classifier Calls", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~\"qwen.*\"}[5m]))", + "refId": "A", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Local Qwen routing-classifier activity, split by successful and failed calls." + }, + { + "id": 22, + "type": "timeseries", + "title": "Routing Overhead p95", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))", + "refId": "A", + "legendFormat": "{{algorithm}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "95th percentile time Switchyard spends selecting a model before the upstream call." + }, + { + "id": 23, + "type": "bargauge", + "title": "Traffic Lanes (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, + "targets": [ + { + "expr": "sort_desc(sum by (lane) (label_replace(increase(switchyard_requests_total{model=~\"(route|worker)/.*\"}[$__range]), \"lane\", \"$1\", \"model\", \"^(route|worker)/.*\")))", + "refId": "A", + "legendFormat": "{{lane}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 50 + }, + { + "color": "dark-orange", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Request volume split between interactive route traffic and durable worker traffic.", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + } + ] + }, + { + "id": 24, + "type": "text", + "title": "Cost Semantics", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "datasource": null, + "options": { + "mode": "markdown", + "content": "Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage." + } + }, + { + "id": 25, + "type": "timeseries", + "title": "Hermes Workload CPU (Attribution Proxy)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "targets": [ + { + "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"hermes\",pod=~\"hermes-(agent|chat-tenant|switchyard|model-gate).*\",container!=\"\",image!=\"\"}[5m]))", + "refId": "A", + "legendFormat": "{{pod}} \u00b7 {{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage." + } + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "1m", + "annotations": { + "list": [] + }, + "schemaVersion": 39, + "style": "dark", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ] +} diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml new file mode 100644 index 00000000..5024ebea --- /dev/null +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -0,0 +1,1612 @@ +# services/monitoring/grafana-dashboard-ai.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-ai + labels: + grafana_dashboard: "1" +data: + atlas-ai.json: | + { + "uid": "atlas-ai", + "title": "Atlas AI Operations", + "folderUid": "atlas-internal", + "editable": true, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Codex Weekly Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 2, + "type": "stat", + "title": "Codex Spark Weekly Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 3, + "type": "stat", + "title": "Claude 5h Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 4, + "type": "stat", + "title": "Claude 7d Remaining", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-orange", + "value": 15 + }, + { + "color": "dark-yellow", + "value": 30 + }, + { + "color": "dark-green", + "value": 50 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 5, + "type": "stat", + "title": "Quota Collectors Healthy", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 2 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto", + "valueSuffix": " / 2" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful latest quota fetches. Providers are polled independently every five minutes." + }, + { + "id": 6, + "type": "stat", + "title": "Oldest Quota Sample", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "expr": "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-green", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 600 + }, + { + "color": "dark-red", + "value": 1200 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Age of the stalest successful provider quota snapshot." + }, + { + "id": 7, + "type": "stat", + "title": "Codex Weekly Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 8, + "type": "stat", + "title": "Claude 5h Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 9, + "type": "stat", + "title": "Claude 7d Reset In", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "targets": [ + { + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 10, + "type": "stat", + "title": "Codex Tokens (Latest Day)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 11, + "type": "stat", + "title": "Codex Tokens (7d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 4 + }, + "targets": [ + { + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "unavailable", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "gray", + "value": null + }, + { + "color": "dark-blue", + "value": 0 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + }, + { + "id": 12, + "type": "stat", + "title": "Switchyard Requests", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "short", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model requests observed by Switchyard in the selected dashboard range." + }, + { + "id": 13, + "type": "timeseries", + "title": "Model Selection Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "expr": "sum by (selected_model) (rate(switchyard_decisions_total[5m]))", + "refId": "A", + "legendFormat": "{{selected_model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort." + }, + { + "id": 14, + "type": "bargauge", + "title": "Provider Selections (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "sort_desc(sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~\"(route|worker)/(codex|claude|local)/.*\"}[$__range]), \"provider\", \"$2\", \"selected_model\", \"^(route|worker)/(codex|claude|local)/.*\")))", + "refId": "A", + "legendFormat": "{{provider}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 50 + }, + { + "color": "dark-orange", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Switchyard selections grouped by provider over the selected dashboard range.", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Token Throughput", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "expr": "sum(rate(switchyard_prompt_tokens_total[5m]))", + "refId": "A", + "legendFormat": "prompt" + }, + { + "expr": "sum(rate(switchyard_cached_tokens_total[5m]))", + "refId": "B", + "legendFormat": "cached" + }, + { + "expr": "sum(rate(switchyard_cache_creation_tokens_total[5m]))", + "refId": "C", + "legendFormat": "cache creation" + }, + { + "expr": "sum(rate(switchyard_reasoning_tokens_total[5m]))", + "refId": "D", + "legendFormat": "reasoning" + }, + { + "expr": "sum(rate(switchyard_completion_tokens_total[5m]))", + "refId": "E", + "legendFormat": "completion" + } + ], + "fieldConfig": { + "defaults": { + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls." + }, + { + "id": 16, + "type": "timeseries", + "title": "Model Call p95 Latency", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))", + "refId": "A", + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "95th percentile upstream latency for each selected model route." + }, + { + "id": 17, + "type": "stat", + "title": "Prompt Cache Share", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 24 + }, + "targets": [ + { + "expr": "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work." + }, + { + "id": 18, + "type": "stat", + "title": "Client Success Rate", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 24 + }, + "targets": [ + { + "expr": "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 99 + } + ] + }, + "unit": "percent", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Successful client-facing Switchyard responses in the selected range." + }, + { + "id": 19, + "type": "stat", + "title": "Classifier Fail-Open (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Local classifier failures that safely fell back to the conservative hosted route." + }, + { + "id": 20, + "type": "stat", + "title": "Upstream Errors (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "targets": [ + { + "expr": "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Hosted model attempts that returned errors in the selected range." + }, + { + "id": 21, + "type": "timeseries", + "title": "Local Classifier Calls", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~\"qwen.*\"}[5m]))", + "refId": "A", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Local Qwen routing-classifier activity, split by successful and failed calls." + }, + { + "id": 22, + "type": "timeseries", + "title": "Routing Overhead p95", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))", + "refId": "A", + "legendFormat": "{{algorithm}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "95th percentile time Switchyard spends selecting a model before the upstream call." + }, + { + "id": 23, + "type": "bargauge", + "title": "Traffic Lanes (Range)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, + "targets": [ + { + "expr": "sort_desc(sum by (lane) (label_replace(increase(switchyard_requests_total{model=~\"(route|worker)/.*\"}[$__range]), \"lane\", \"$1\", \"model\", \"^(route|worker)/.*\")))", + "refId": "A", + "legendFormat": "{{lane}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 50 + }, + { + "color": "dark-orange", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Request volume split between interactive route traffic and durable worker traffic.", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + } + ] + }, + { + "id": 24, + "type": "text", + "title": "Cost Semantics", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "datasource": null, + "options": { + "mode": "markdown", + "content": "Codex and Claude currently run through first-party subscription OAuth lanes, so providers expose quota utilization rather than per-call dollar invoices. This dashboard does not invent API costs from tokens. If a metered API-key lane is added, its provider billing metric should be displayed separately from subscription usage." + } + }, + { + "id": 25, + "type": "timeseries", + "title": "Hermes Workload CPU (Attribution Proxy)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "targets": [ + { + "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"hermes\",pod=~\"hermes-(agent|chat-tenant|switchyard|model-gate).*\",container!=\"\",image!=\"\"}[5m]))", + "refId": "A", + "legendFormat": "{{pod}} \u00b7 {{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage." + } + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "1m", + "annotations": { + "list": [] + }, + "schemaVersion": 39, + "style": "dark", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ] + } diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index 4ab92c5a..ad30aa34 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -622,6 +622,15 @@ spec: updateIntervalSeconds: 10 options: path: /var/lib/grafana/dashboards/testing-public + - name: ai + orgId: 1 + folder: Atlas Internal + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards/ai - name: power orgId: 1 folder: Atlas Internal @@ -651,6 +660,7 @@ spec: mail: grafana-dashboard-mail testing: grafana-dashboard-testing testing-public: grafana-dashboard-testing + ai: grafana-dashboard-ai power: grafana-dashboard-power cassandra: grafana-dashboard-cassandra extraConfigmapMounts: diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 1c8f537f..45627b50 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -18,6 +18,7 @@ resources: - grafana-dashboard-power.yaml - grafana-dashboard-mail.yaml - grafana-dashboard-testing.yaml + - grafana-dashboard-ai.yaml - vmalert-atlas-availability.yaml - availability-backfill-v4-job.yaml - availability-daily-backfill-v4-job.yaml diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py new file mode 100644 index 00000000..ed9dbeae --- /dev/null +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -0,0 +1,65 @@ +"""Contract checks for the internal Atlas AI operations dashboard.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py" + + +def load_module(): + """Load the dashboard generator without invoking its CLI.""" + spec = importlib.util.spec_from_file_location("dashboards_render_atlas_ai", SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): + mod = load_module() + dashboard = mod.build_ai_dashboard() + panels = {panel["title"]: panel for panel in dashboard["panels"]} + expressions = "\n".join( + target.get("expr", "") + for panel in dashboard["panels"] + for target in panel.get("targets", []) + ) + + assert dashboard["uid"] == "atlas-ai" + assert dashboard["folderUid"] == mod.PRIVATE_FOLDER + assert dashboard["refresh"] == "1m" + assert "Codex Weekly Remaining" in panels + assert "Claude 5h Remaining" in panels + assert "Claude 7d Remaining" in panels + assert "Provider Selections (Range)" in panels + assert "Local Classifier Calls" in panels + assert "Hermes Workload CPU (Attribution Proxy)" in panels + assert "atlas_ai_quota_remaining_percent" in expressions + assert "atlas_ai_quota_reset_timestamp_seconds" in expressions + assert "switchyard_decisions_total" in expressions + assert "switchyard_cached_tokens_total" in expressions + assert "switchyard_model_call_latency_ms_bucket" in expressions + assert "/status" not in expressions + assert all( + panel.get("description") or panel["type"] == "text" + for panel in dashboard["panels"] + ) + + +def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution(): + mod = load_module() + dashboard = mod.build_ai_dashboard() + panels = {panel["title"]: panel for panel in dashboard["panels"]} + + cost_text = panels["Cost Semantics"]["options"]["content"] + proxy_description = panels["Hermes Workload CPU (Attribution Proxy)"]["description"] + assert "does not invent API costs" in cost_text + assert "subscription" in cost_text + assert "proxy" in proxy_description + assert "not tenant-slot token labels" in proxy_description diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py new file mode 100644 index 00000000..9b78b7f3 --- /dev/null +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -0,0 +1,175 @@ +"""Tests for the credential-safe Atlas AI usage exporter.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from datetime import date +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" + + +def load_module(): + """Load the standalone exporter script as a test module.""" + spec = importlib.util.spec_from_file_location("ai_usage_exporter", SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_codex_payloads_include_real_windows_and_bounded_token_periods(): + mod = load_module() + rate = { + "rateLimitsByLimitId": { + "codex": { + "primary": { + "usedPercent": 41, + "windowDurationMins": 10080, + "resetsAt": 1_800_000_000, + } + }, + "opaque_backend_id": { + "limitName": "GPT-5.3-Codex-Spark", + "primary": { + "usedPercent": 0, + "windowDurationMins": 10080, + "resetsAt": 1_800_100_000, + }, + }, + } + } + usage = { + "summary": {"lifetimeTokens": 1234, "peakDailyTokens": 500}, + "dailyUsageBuckets": [ + {"startDate": "2026-08-16", "tokens": 100}, + {"startDate": "2026-08-15", "tokens": 50}, + {"startDate": "2026-07-01", "tokens": 900}, + ], + } + + samples = mod.parse_codex_payloads(rate, usage, today=date(2026, 8, 16)) + values = {(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples} + + codex_labels = ( + ("limit", "codex"), + ("provider", "openai"), + ("window", "seven_day"), + ) + spark_labels = ( + ("limit", "gpt-5-3-codex-spark"), + ("provider", "openai"), + ("window", "seven_day"), + ) + assert values[("atlas_ai_quota_used_percent", codex_labels)] == 41 + assert values[("atlas_ai_quota_remaining_percent", codex_labels)] == 59 + assert values[("atlas_ai_quota_used_percent", spark_labels)] == 0 + assert values[ + ( + "atlas_ai_account_tokens", + (("period", "latest_day"), ("provider", "openai")), + ) + ] == 100 + assert values[ + ( + "atlas_ai_account_tokens", + (("period", "seven_day"), ("provider", "openai")), + ) + ] == 150 + + +def test_claude_payload_uses_only_supported_quota_fields(): + mod = load_module() + secret = "secret-access-token-must-not-leak" + payload = { + "five_hour": { + "utilization": 16.0, + "resets_at": "2026-08-16T08:39:59+00:00", + "unexpected_secret": secret, + }, + "seven_day": { + "utilization": 26.0, + "resets_at": "2026-08-20T17:59:59+00:00", + }, + "internal_bucket": {"utilization": 99, "resets_at": secret}, + "extra_usage": {"is_enabled": False, "used_credits": secret}, + } + + samples = mod.parse_claude_payload(payload) + assert {item.labels["window"] for item in samples if "window" in item.labels} == { + "five_hour", + "seven_day", + } + assert any( + item.name == "atlas_ai_quota_remaining_percent" + and item.labels["window"] == "seven_day" + and item.value == 74 + for item in samples + ) + assert secret not in repr(samples) + + +def test_render_reports_failure_and_freshness_without_logging_credentials(): + mod = load_module() + collector = mod.Collector() + state = collector._providers["openai"] + state.samples = [ + mod.Sample( + "atlas_ai_quota_used_percent", + {"provider": "openai", "limit": "codex", "window": "seven_day"}, + 42, + ) + ] + state.fetch_success = False + state.last_attempt = 200 + state.last_success = 100 + rendered = collector.render().decode() + + assert 'atlas_ai_quota_fetch_success{provider="openai"} 0' in rendered + assert 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' in rendered + assert 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered + assert "accessToken" not in rendered + assert "refreshToken" not in rendered + + +def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): + mod = load_module() + request_log = tmp_path / "requests.jsonl" + mock = tmp_path / "codex-mock" + mock.write_text( + """#!/usr/bin/env python3 +import json, os, sys +messages = [json.loads(sys.stdin.readline()) for _ in range(4)] +with open(os.environ['REQUEST_LOG'], 'w') as handle: + for message in messages: + handle.write(json.dumps(message) + '\\n') +responses = { + 1: {}, + 2: {'rateLimits': {'primary': {'usedPercent': 10}}}, + 3: {'summary': {}, 'dailyUsageBuckets': []}, +} +for response_id, result in responses.items(): + print(json.dumps({'id': response_id, 'result': result}), flush=True) +""" + ) + mock.chmod(0o755) + monkeypatch.setattr(mod, "CODEX_BIN", str(mock)) + monkeypatch.setenv("REQUEST_LOG", str(request_log)) + + rate, usage = mod.query_codex(timeout=5) + requests = [json.loads(line) for line in request_log.read_text().splitlines()] + + assert rate["rateLimits"]["primary"]["usedPercent"] == 10 + assert usage["summary"] == {} + assert [request["method"] for request in requests] == [ + "initialize", + "initialized", + "account/rateLimits/read", + "account/usage/read", + ] + assert all("/status" not in json.dumps(request) for request in requests)