diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index adac0d25..aa06ac1c 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5219,13 +5219,22 @@ def build_ai_dashboard(): decimals=1, thresholds=thresholds, instant=True, - description="Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy.", + description="Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale.", ) panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping return panel - openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1' - anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1' + def snapshot_fresh(provider): + """Require authenticated access and a successful snapshot under 20m old.""" + authenticated = f'atlas_ai_provider_authenticated{{provider="{provider}"}} == 1' + last_success = ( + "time() - atlas_ai_quota_last_success_timestamp_seconds" + f'{{provider="{provider}"}} < 1200' + ) + return f"({authenticated}) and on(provider) ({last_success})" + + openai_snapshot = snapshot_fresh("openai") + anthropic_snapshot = snapshot_fresh("anthropic") anthropic_authenticated = ( 'atlas_ai_provider_authenticated{provider="anthropic"} == 1' ) @@ -5251,26 +5260,26 @@ def build_ai_dashboard(): quota_stat( 1, "Codex Weekly Remaining", - f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_ok})', + f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 0, "y": 0}, ), quota_stat( 2, "Codex Spark Weekly Remaining", - f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_ok})', + f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 4, "y": 0}, ), quota_stat( 3, "Claude 5h Remaining", - f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})', + f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_snapshot})', {"h": 4, "w": 4, "x": 8, "y": 0}, authenticated_expr=anthropic_authenticated, ), quota_stat( 4, "Claude 7d Remaining", - f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})', + f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_snapshot})', {"h": 4, "w": 4, "x": 12, "y": 0}, authenticated_expr=anthropic_authenticated, ), @@ -5312,7 +5321,7 @@ def build_ai_dashboard(): quota_stat( 7, "Codex Weekly Reset In", - f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_ok})', + f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 0, "y": 4}, unit="s", ), @@ -5339,14 +5348,14 @@ def build_ai_dashboard(): quota_stat( 10, "Codex Tokens (Latest Day)", - f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_ok})', + f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 12, "y": 4}, unit="short", ), quota_stat( 11, "Codex Tokens (7d)", - f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_ok})', + f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_snapshot})', {"h": 4, "w": 4, "x": 16, "y": 4}, unit="short", ), diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 50115045..2cea82a7 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260822-claude-setup-token-v2" + ai.bstein.dev/config-rev: "20260823-ai-quota-stability" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" @@ -1033,6 +1033,7 @@ spec: - {name: HOME, value: /tmp} - {name: CODEX_HOME, value: /runtime-access/codex} - {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex} + - {name: ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS, value: "45"} - {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json} - {name: ATLAS_AI_PROVIDER_HEALTH_ROOT, value: /provider-health} - {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"} diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index e5a4caef..7bd7a638 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -105,6 +105,7 @@ configMapGenerator: - ai_usage_codex.py=scripts/ai_usage_codex.py - ai_usage_exporter.py=scripts/ai_usage_exporter.py - ai_usage_http.py=scripts/ai_usage_http.py + - ai_usage_polling.py=scripts/ai_usage_polling.py - claude=scripts/claude - claude_oauth_exec=scripts/claude_oauth_exec - claude_command_policy.py=scripts/claude_command_policy.py diff --git a/services/hermes/scripts/ai_usage_codex.py b/services/hermes/scripts/ai_usage_codex.py index 6214cbf5..5db12467 100644 --- a/services/hermes/scripts/ai_usage_codex.py +++ b/services/hermes/scripts/ai_usage_codex.py @@ -13,6 +13,10 @@ from typing import Any CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") CODEX_CLEANUP_TIMEOUT_SECONDS = 5 +CODEX_QUERY_TIMEOUT_SECONDS = min( + 60.0, + max(20.0, float(os.environ.get("ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS", "45"))), +) def _close_stream(stream: Any) -> bool: @@ -72,8 +76,9 @@ def _cleanup_codex_process( return cleanup_ok -def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: +def query_codex(timeout: float | None = None) -> tuple[dict[str, Any], dict[str, Any]]: """Read Codex account quota and usage through its structured app-server protocol.""" + timeout = CODEX_QUERY_TIMEOUT_SECONDS if timeout is None else timeout process = subprocess.Popen( [CODEX_BIN, "app-server", "--stdio"], stdin=subprocess.PIPE, diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index 0b93b686..13fdb167 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -15,6 +15,7 @@ from urllib.request import Request, urlopen import ai_usage_codex as codex_query import ai_usage_http as http_engine +import ai_usage_polling as polling_engine CLAUDE_CREDENTIALS = Path( @@ -50,10 +51,6 @@ METRIC_HELP = { "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.", "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.", } -POLL_STARTUP_GRACE_SECONDS = 45 -POLL_PROGRESS_BUDGET_SECONDS = 60 - - class QuotaNotExposed(RuntimeError): """The active provider credential cannot expose account quota.""" @@ -434,101 +431,9 @@ class Collector: return ("\n".join(lines) + "\n").encode("utf-8") -class PollingEngine: - """Run isolated provider polls and track bounded forward progress.""" - - def __init__( - self, - collector: Collector, - *, - interval: float, - startup_grace: float = POLL_STARTUP_GRACE_SECONDS, - progress_timeout: float | None = None, - clock: Any = time.monotonic, - ) -> None: - self.collector = collector - self.interval = interval - self.startup_grace = startup_grace - self.progress_timeout = ( - interval + POLL_PROGRESS_BUDGET_SECONDS - if progress_timeout is None - else progress_timeout - ) - self._clock = clock - self._started_at = clock() - self._last_progress: float | None = None - self._progress_lock = threading.Lock() - self._thread: threading.Thread | None = None - - def _mark_progress(self) -> None: - """Record completion of one bounded provider attempt.""" - with self._progress_lock: - self._last_progress = self._clock() - - def poll_once(self) -> None: - """Refresh every provider even if another provider fails unexpectedly.""" - for provider in ("openai", "anthropic"): - started = time.time() - monotonic_started = time.monotonic() - try: - self.collector.refresh_provider(provider) - except Exception as error: - print( - f"{provider} quota collection isolated: {type(error).__name__}", - flush=True, - ) - try: - self.collector.record_failure( - provider, - started=started, - monotonic_started=monotonic_started, - ) - except Exception as record_error: - print( - f"{provider} quota failure accounting deferred: " - f"{type(record_error).__name__}", - flush=True, - ) - finally: - self._mark_progress() - - def run(self) -> None: - """Poll forever without allowing a cycle-level exception to stop the thread.""" - while True: - try: - self.poll_once() - except Exception as error: - print( - f"quota polling cycle deferred: {type(error).__name__}", flush=True - ) - time.sleep(self.interval) - - def start(self) -> None: - """Start the daemon poller exactly once.""" - if self._thread is not None: - return - self._thread = threading.Thread( - target=self.run, - name="ai-usage-poller", - daemon=True, - ) - self._thread.start() - - def is_healthy(self) -> bool: - """Report thread liveness and progress, independent of provider success.""" - thread = self._thread - if thread is None or not thread.is_alive(): - return False - now = self._clock() - with self._progress_lock: - last_progress = self._last_progress - if last_progress is None: - return now - self._started_at <= self.startup_grace - return now - last_progress <= self.progress_timeout - - make_handler = http_engine.make_handler Server = http_engine.Server +PollingEngine = polling_engine.PollingEngine def main() -> int: diff --git a/services/hermes/scripts/ai_usage_polling.py b/services/hermes/scripts/ai_usage_polling.py new file mode 100644 index 00000000..2825a057 --- /dev/null +++ b/services/hermes/scripts/ai_usage_polling.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Bounded polling lifecycle and progress health for the AI usage exporter.""" + +from __future__ import annotations + +import threading +import time +from typing import Any + + +POLL_STARTUP_GRACE_SECONDS = 45 +POLL_PROGRESS_BUDGET_SECONDS = 120 + + +class PollingEngine: + """Run isolated provider polls and track bounded forward progress.""" + + def __init__( + self, + collector: Any, + *, + interval: float, + startup_grace: float = POLL_STARTUP_GRACE_SECONDS, + progress_timeout: float | None = None, + clock: Any = time.monotonic, + ) -> None: + self.collector = collector + self.interval = interval + self.startup_grace = startup_grace + self.progress_timeout = ( + interval + POLL_PROGRESS_BUDGET_SECONDS + if progress_timeout is None + else progress_timeout + ) + self._clock = clock + self._started_at = clock() + self._last_progress: float | None = None + self._progress_lock = threading.Lock() + self._thread: threading.Thread | None = None + + def _mark_progress(self) -> None: + """Record completion of one bounded provider attempt.""" + with self._progress_lock: + self._last_progress = self._clock() + + def poll_once(self) -> None: + """Refresh every provider even if another provider fails unexpectedly.""" + for provider in ("openai", "anthropic"): + started = time.time() + monotonic_started = time.monotonic() + try: + self.collector.refresh_provider(provider) + except Exception as error: + print( + f"{provider} quota collection isolated: {type(error).__name__}", + flush=True, + ) + try: + self.collector.record_failure( + provider, + started=started, + monotonic_started=monotonic_started, + ) + except Exception as record_error: + print( + f"{provider} quota failure accounting deferred: " + f"{type(record_error).__name__}", + flush=True, + ) + finally: + self._mark_progress() + + def run(self) -> None: + """Poll forever without allowing a cycle-level exception to stop the thread.""" + while True: + try: + self.poll_once() + except Exception as error: + print( + f"quota polling cycle deferred: {type(error).__name__}", flush=True + ) + time.sleep(self.interval) + + def start(self) -> None: + """Start the daemon poller exactly once.""" + if self._thread is not None: + return + self._thread = threading.Thread( + target=self.run, + name="ai-usage-poller", + daemon=True, + ) + self._thread.start() + + def is_healthy(self) -> bool: + """Report thread liveness and progress, independent of provider success.""" + thread = self._thread + if thread is None or not thread.is_alive(): + return False + now = self._clock() + with self._progress_lock: + last_progress = self._last_progress + if last_progress is None: + return now - self._started_at <= self.startup_grace + return now - last_progress <= self.progress_timeout diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index f745d809..75be5652 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -91,7 +91,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 2, @@ -109,7 +109,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -180,7 +180,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 3, @@ -198,7 +198,7 @@ }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -269,7 +269,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 4, @@ -287,7 +287,7 @@ }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -358,7 +358,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 5, @@ -513,7 +513,7 @@ }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -572,7 +572,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 8, @@ -716,7 +716,7 @@ }, "targets": [ { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -775,7 +775,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 11, @@ -793,7 +793,7 @@ }, "targets": [ { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -852,7 +852,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 12, diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index ad209720..f24f419c 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -100,7 +100,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 2, @@ -118,7 +118,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -189,7 +189,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 3, @@ -207,7 +207,7 @@ data: }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -278,7 +278,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 4, @@ -296,7 +296,7 @@ data: }, "targets": [ { - "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -367,7 +367,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 5, @@ -522,7 +522,7 @@ data: }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -581,7 +581,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 8, @@ -725,7 +725,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -784,7 +784,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 11, @@ -802,7 +802,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)", + "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)", "refId": "A", "instant": true } @@ -861,7 +861,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." + "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale." }, { "id": 12, diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index 455b6ead..afb7d8e2 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -57,6 +57,26 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): ) +def test_ai_quota_panels_retain_a_bounded_last_good_snapshot(): + """One transient collector miss must not blank every account panel.""" + mod = load_module() + panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} + + for title in ( + "Codex Weekly Remaining", + "Codex Spark Weekly Remaining", + "Codex Weekly Reset In", + "Codex Tokens (Latest Day)", + "Codex Tokens (7d)", + "Claude 5h Remaining", + "Claude 7d Remaining", + ): + expression = panels[title]["targets"][0]["expr"] + assert "atlas_ai_quota_last_success_timestamp_seconds" in expression + assert "1200" in expression + assert "atlas_ai_quota_fetch_success" not in expression + + def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution(): mod = load_module() dashboard = mod.build_ai_dashboard() diff --git a/testing/tests/test_hermes_ai_usage_codex.py b/testing/tests/test_hermes_ai_usage_codex.py index a77525bf..b6c5ccf1 100644 --- a/testing/tests/test_hermes_ai_usage_codex.py +++ b/testing/tests/test_hermes_ai_usage_codex.py @@ -28,6 +28,14 @@ def load_module(): return module +def test_default_query_deadline_has_margin_for_live_app_server_jitter(monkeypatch): + """The default must not sit on the live app-server's common 20s boundary.""" + monkeypatch.setenv("ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS", "45") + mod = load_module() + + assert mod.CODEX_QUERY_TIMEOUT_SECONDS == 45 + + def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): mod = load_module() request_log = tmp_path / "requests.jsonl" diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index c33654dc..41400f40 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -8,9 +8,12 @@ from datetime import date from io import BytesIO from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" +HERMES = ROOT / "services/hermes" def load_module(): @@ -218,6 +221,8 @@ def test_health_tracks_poller_liveness_and_progress_not_provider_success(): now = [100.0] collector = mod.Collector() collector._providers["openai"].fetch_success = False + default_engine = mod.PollingEngine(collector, interval=300, clock=lambda: now[0]) + assert default_engine.progress_timeout == 420 engine = mod.PollingEngine( collector, interval=60, @@ -249,6 +254,21 @@ def test_health_tracks_poller_liveness_and_progress_not_provider_success(): assert not engine.is_healthy() +def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module(): + """Flux must restart the exporter with every reliability input mounted.""" + deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + containers = deployment["spec"]["template"]["spec"]["containers"] + exporter = next(item for item in containers if item["name"] == "ai-usage-exporter") + environment = {item["name"]: item["value"] for item in exporter["env"]} + + assert annotations["ai.bstein.dev/config-rev"] == "20260823-ai-quota-stability" + assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45" + assert "ai_usage_polling.py=scripts/ai_usage_polling.py" in ( + HERMES / "kustomization.yaml" + ).read_text() + + def test_health_endpoint_uses_poller_state_and_ignores_provider_failure(): mod = load_module() collector = mod.Collector()