hermes: stabilize AI quota collection

scripts/render/dashboards_render_atlas.py

@@ -5219,13 +5219,22 @@ def build_ai_dashboard():
             decimals=1,
             thresholds=thresholds,
             instant=True,
-            description="Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy.",
+            description="Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale.",
         )
         panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping
         return panel
 
-    openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1'
+    def snapshot_fresh(provider):
-    anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1'
+        """Require authenticated access and a successful snapshot under 20m old."""
+        authenticated = f'atlas_ai_provider_authenticated{{provider="{provider}"}} == 1'
+        last_success = (
+            "time() - atlas_ai_quota_last_success_timestamp_seconds"
+            f'{{provider="{provider}"}} < 1200'
+        )
+        return f"({authenticated}) and on(provider) ({last_success})"
+
+    openai_snapshot = snapshot_fresh("openai")
+    anthropic_snapshot = snapshot_fresh("anthropic")
     anthropic_authenticated = (
         'atlas_ai_provider_authenticated{provider="anthropic"} == 1'
     )
@@ -5251,26 +5260,26 @@ def build_ai_dashboard():
         quota_stat(
             1,
             "Codex Weekly Remaining",
-            f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_ok})',
+            f'{quota}{{provider="openai",limit="codex",window="seven_day"}} and on(provider) ({openai_snapshot})',
             {"h": 4, "w": 4, "x": 0, "y": 0},
         ),
         quota_stat(
             2,
             "Codex Spark Weekly Remaining",
-            f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_ok})',
+            f'{quota}{{provider="openai",limit="gpt-5-3-codex-spark",window="seven_day"}} and on(provider) ({openai_snapshot})',
             {"h": 4, "w": 4, "x": 4, "y": 0},
         ),
         quota_stat(
             3,
             "Claude 5h Remaining",
-            f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})',
+            f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_snapshot})',
             {"h": 4, "w": 4, "x": 8, "y": 0},
             authenticated_expr=anthropic_authenticated,
         ),
         quota_stat(
             4,
             "Claude 7d Remaining",
-            f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})',
+            f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_snapshot})',
             {"h": 4, "w": 4, "x": 12, "y": 0},
             authenticated_expr=anthropic_authenticated,
         ),
@@ -5312,7 +5321,7 @@ def build_ai_dashboard():
         quota_stat(
             7,
             "Codex Weekly Reset In",
-            f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_ok})',
+            f'clamp_min({reset}{{provider="openai",limit="codex",window="seven_day"}} - time(), 0) and on(provider) ({openai_snapshot})',
             {"h": 4, "w": 4, "x": 0, "y": 4},
             unit="s",
         ),
@@ -5339,14 +5348,14 @@ def build_ai_dashboard():
         quota_stat(
             10,
             "Codex Tokens (Latest Day)",
-            f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_ok})',
+            f'atlas_ai_account_tokens{{provider="openai",period="latest_day"}} and on(provider) ({openai_snapshot})',
             {"h": 4, "w": 4, "x": 12, "y": 4},
             unit="short",
         ),
         quota_stat(
             11,
             "Codex Tokens (7d)",
-            f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_ok})',
+            f'atlas_ai_account_tokens{{provider="openai",period="seven_day"}} and on(provider) ({openai_snapshot})',
             {"h": 4, "w": 4, "x": 16, "y": 4},
             unit="short",
         ),

services/hermes/agent-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
-        ai.bstein.dev/config-rev: "20260822-claude-setup-token-v2"
+        ai.bstein.dev/config-rev: "20260823-ai-quota-stability"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"
@@ -1033,6 +1033,7 @@ spec:
             - {name: HOME, value: /tmp}
             - {name: CODEX_HOME, value: /runtime-access/codex}
             - {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex}
+            - {name: ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS, value: "45"}
             - {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json}
             - {name: ATLAS_AI_PROVIDER_HEALTH_ROOT, value: /provider-health}
             - {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"}

services/hermes/kustomization.yaml

@@ -105,6 +105,7 @@ configMapGenerator:
       - ai_usage_codex.py=scripts/ai_usage_codex.py
       - ai_usage_exporter.py=scripts/ai_usage_exporter.py
       - ai_usage_http.py=scripts/ai_usage_http.py
+      - ai_usage_polling.py=scripts/ai_usage_polling.py
       - claude=scripts/claude
       - claude_oauth_exec=scripts/claude_oauth_exec
       - claude_command_policy.py=scripts/claude_command_policy.py

services/hermes/scripts/ai_usage_codex.py

@@ -13,6 +13,10 @@ from typing import Any
 CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
 CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
 CODEX_CLEANUP_TIMEOUT_SECONDS = 5
+CODEX_QUERY_TIMEOUT_SECONDS = min(
+    60.0,
+    max(20.0, float(os.environ.get("ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS", "45"))),
+)
 
 
 def _close_stream(stream: Any) -> bool:
@@ -72,8 +76,9 @@ def _cleanup_codex_process(
     return cleanup_ok
 
 
-def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
+def query_codex(timeout: float | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
     """Read Codex account quota and usage through its structured app-server protocol."""
+    timeout = CODEX_QUERY_TIMEOUT_SECONDS if timeout is None else timeout
     process = subprocess.Popen(
         [CODEX_BIN, "app-server", "--stdio"],
         stdin=subprocess.PIPE,

services/hermes/scripts/ai_usage_exporter.py

@@ -15,6 +15,7 @@ from urllib.request import Request, urlopen
 
 import ai_usage_codex as codex_query
 import ai_usage_http as http_engine
+import ai_usage_polling as polling_engine
 
 
 CLAUDE_CREDENTIALS = Path(
@@ -50,10 +51,6 @@ METRIC_HELP = {
     "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.",
     "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.",
 }
-POLL_STARTUP_GRACE_SECONDS = 45
-POLL_PROGRESS_BUDGET_SECONDS = 60
-
-
 class QuotaNotExposed(RuntimeError):
     """The active provider credential cannot expose account quota."""
 
@@ -434,101 +431,9 @@ class Collector:
         return ("\n".join(lines) + "\n").encode("utf-8")
 
 
-class PollingEngine:
-    """Run isolated provider polls and track bounded forward progress."""
-
-    def __init__(
-        self,
-        collector: Collector,
-        *,
-        interval: float,
-        startup_grace: float = POLL_STARTUP_GRACE_SECONDS,
-        progress_timeout: float | None = None,
-        clock: Any = time.monotonic,
-    ) -> None:
-        self.collector = collector
-        self.interval = interval
-        self.startup_grace = startup_grace
-        self.progress_timeout = (
-            interval + POLL_PROGRESS_BUDGET_SECONDS
-            if progress_timeout is None
-            else progress_timeout
-        )
-        self._clock = clock
-        self._started_at = clock()
-        self._last_progress: float | None = None
-        self._progress_lock = threading.Lock()
-        self._thread: threading.Thread | None = None
-
-    def _mark_progress(self) -> None:
-        """Record completion of one bounded provider attempt."""
-        with self._progress_lock:
-            self._last_progress = self._clock()
-
-    def poll_once(self) -> None:
-        """Refresh every provider even if another provider fails unexpectedly."""
-        for provider in ("openai", "anthropic"):
-            started = time.time()
-            monotonic_started = time.monotonic()
-            try:
-                self.collector.refresh_provider(provider)
-            except Exception as error:
-                print(
-                    f"{provider} quota collection isolated: {type(error).__name__}",
-                    flush=True,
-                )
-                try:
-                    self.collector.record_failure(
-                        provider,
-                        started=started,
-                        monotonic_started=monotonic_started,
-                    )
-                except Exception as record_error:
-                    print(
-                        f"{provider} quota failure accounting deferred: "
-                        f"{type(record_error).__name__}",
-                        flush=True,
-                    )
-            finally:
-                self._mark_progress()
-
-    def run(self) -> None:
-        """Poll forever without allowing a cycle-level exception to stop the thread."""
-        while True:
-            try:
-                self.poll_once()
-            except Exception as error:
-                print(
-                    f"quota polling cycle deferred: {type(error).__name__}", flush=True
-                )
-            time.sleep(self.interval)
-
-    def start(self) -> None:
-        """Start the daemon poller exactly once."""
-        if self._thread is not None:
-            return
-        self._thread = threading.Thread(
-            target=self.run,
-            name="ai-usage-poller",
-            daemon=True,
-        )
-        self._thread.start()
-
-    def is_healthy(self) -> bool:
-        """Report thread liveness and progress, independent of provider success."""
-        thread = self._thread
-        if thread is None or not thread.is_alive():
-            return False
-        now = self._clock()
-        with self._progress_lock:
-            last_progress = self._last_progress
-        if last_progress is None:
-            return now - self._started_at <= self.startup_grace
-        return now - last_progress <= self.progress_timeout
-
-
 make_handler = http_engine.make_handler
 Server = http_engine.Server
+PollingEngine = polling_engine.PollingEngine
 
 
 def main() -> int:

services/hermes/scripts/ai_usage_polling.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Bounded polling lifecycle and progress health for the AI usage exporter."""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any
+
+
+POLL_STARTUP_GRACE_SECONDS = 45
+POLL_PROGRESS_BUDGET_SECONDS = 120
+
+
+class PollingEngine:
+    """Run isolated provider polls and track bounded forward progress."""
+
+    def __init__(
+        self,
+        collector: Any,
+        *,
+        interval: float,
+        startup_grace: float = POLL_STARTUP_GRACE_SECONDS,
+        progress_timeout: float | None = None,
+        clock: Any = time.monotonic,
+    ) -> None:
+        self.collector = collector
+        self.interval = interval
+        self.startup_grace = startup_grace
+        self.progress_timeout = (
+            interval + POLL_PROGRESS_BUDGET_SECONDS
+            if progress_timeout is None
+            else progress_timeout
+        )
+        self._clock = clock
+        self._started_at = clock()
+        self._last_progress: float | None = None
+        self._progress_lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+
+    def _mark_progress(self) -> None:
+        """Record completion of one bounded provider attempt."""
+        with self._progress_lock:
+            self._last_progress = self._clock()
+
+    def poll_once(self) -> None:
+        """Refresh every provider even if another provider fails unexpectedly."""
+        for provider in ("openai", "anthropic"):
+            started = time.time()
+            monotonic_started = time.monotonic()
+            try:
+                self.collector.refresh_provider(provider)
+            except Exception as error:
+                print(
+                    f"{provider} quota collection isolated: {type(error).__name__}",
+                    flush=True,
+                )
+                try:
+                    self.collector.record_failure(
+                        provider,
+                        started=started,
+                        monotonic_started=monotonic_started,
+                    )
+                except Exception as record_error:
+                    print(
+                        f"{provider} quota failure accounting deferred: "
+                        f"{type(record_error).__name__}",
+                        flush=True,
+                    )
+            finally:
+                self._mark_progress()
+
+    def run(self) -> None:
+        """Poll forever without allowing a cycle-level exception to stop the thread."""
+        while True:
+            try:
+                self.poll_once()
+            except Exception as error:
+                print(
+                    f"quota polling cycle deferred: {type(error).__name__}", flush=True
+                )
+            time.sleep(self.interval)
+
+    def start(self) -> None:
+        """Start the daemon poller exactly once."""
+        if self._thread is not None:
+            return
+        self._thread = threading.Thread(
+            target=self.run,
+            name="ai-usage-poller",
+            daemon=True,
+        )
+        self._thread.start()
+
+    def is_healthy(self) -> bool:
+        """Report thread liveness and progress, independent of provider success."""
+        thread = self._thread
+        if thread is None or not thread.is_alive():
+            return False
+        now = self._clock()
+        with self._progress_lock:
+            last_progress = self._last_progress
+        if last_progress is None:
+            return now - self._started_at <= self.startup_grace
+        return now - last_progress <= self.progress_timeout

services/monitoring/dashboards/atlas-ai.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+          "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -91,7 +91,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 2,
@@ -109,7 +109,7 @@
       },
       "targets": [
         {
-          "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+          "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -180,7 +180,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 3,
@@ -198,7 +198,7 @@
       },
       "targets": [
         {
-          "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+          "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -269,7 +269,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 4,
@@ -287,7 +287,7 @@
       },
       "targets": [
         {
-          "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+          "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -358,7 +358,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 5,
@@ -513,7 +513,7 @@
       },
       "targets": [
         {
-          "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+          "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -572,7 +572,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 8,
@@ -716,7 +716,7 @@
       },
       "targets": [
         {
-          "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+          "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -775,7 +775,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 11,
@@ -793,7 +793,7 @@
       },
       "targets": [
         {
-          "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+          "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
           "refId": "A",
           "instant": true
         }
@@ -852,7 +852,7 @@
         },
         "textMode": "value"
       },
-      "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+      "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
     },
     {
       "id": 12,

services/monitoring/grafana-dashboard-ai.yaml

@@ -29,7 +29,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+              "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -100,7 +100,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 2,
@@ -118,7 +118,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+              "expr": "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -189,7 +189,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 3,
@@ -207,7 +207,7 @@ data:
           },
           "targets": [
             {
-              "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+              "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -278,7 +278,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 4,
@@ -296,7 +296,7 @@ data:
           },
           "targets": [
             {
-              "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
+              "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"anthropic\"} < 1200))) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -367,7 +367,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 5,
@@ -522,7 +522,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+              "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -581,7 +581,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 8,
@@ -725,7 +725,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+              "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -784,7 +784,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 11,
@@ -802,7 +802,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)",
+              "expr": "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) ((atlas_ai_provider_authenticated{provider=\"openai\"} == 1) and on(provider) (time() - atlas_ai_quota_last_success_timestamp_seconds{provider=\"openai\"} < 1200))) or on() vector(-1)",
               "refId": "A",
               "instant": true
             }
@@ -861,7 +861,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
+          "description": "Live first-party CLI account telemetry. A last-good snapshot remains visible for up to 20 minutes through a transient fetch miss; its age is shown separately. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means access is unhealthy or the snapshot is stale."
         },
         {
           "id": 12,

testing/tests/test_atlas_ai_dashboard.py

@@ -57,6 +57,26 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
     )
 
 
+def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
+    """One transient collector miss must not blank every account panel."""
+    mod = load_module()
+    panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
+
+    for title in (
+        "Codex Weekly Remaining",
+        "Codex Spark Weekly Remaining",
+        "Codex Weekly Reset In",
+        "Codex Tokens (Latest Day)",
+        "Codex Tokens (7d)",
+        "Claude 5h Remaining",
+        "Claude 7d Remaining",
+    ):
+        expression = panels[title]["targets"][0]["expr"]
+        assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
+        assert "1200" in expression
+        assert "atlas_ai_quota_fetch_success" not in expression
+
+
 def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution():
     mod = load_module()
     dashboard = mod.build_ai_dashboard()

testing/tests/test_hermes_ai_usage_codex.py

@@ -28,6 +28,14 @@ def load_module():
     return module
 
 
+def test_default_query_deadline_has_margin_for_live_app_server_jitter(monkeypatch):
+    """The default must not sit on the live app-server's common 20s boundary."""
+    monkeypatch.setenv("ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS", "45")
+    mod = load_module()
+
+    assert mod.CODEX_QUERY_TIMEOUT_SECONDS == 45
+
+
 def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch):
     mod = load_module()
     request_log = tmp_path / "requests.jsonl"

testing/tests/test_hermes_ai_usage_exporter.py

@@ -8,9 +8,12 @@ from datetime import date
 from io import BytesIO
 from pathlib import Path
 
+import yaml
+
 
 ROOT = Path(__file__).resolve().parents[2]
 SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py"
+HERMES = ROOT / "services/hermes"
 
 
 def load_module():
@@ -218,6 +221,8 @@ def test_health_tracks_poller_liveness_and_progress_not_provider_success():
     now = [100.0]
     collector = mod.Collector()
     collector._providers["openai"].fetch_success = False
+    default_engine = mod.PollingEngine(collector, interval=300, clock=lambda: now[0])
+    assert default_engine.progress_timeout == 420
     engine = mod.PollingEngine(
         collector,
         interval=60,
@@ -249,6 +254,21 @@ def test_health_tracks_poller_liveness_and_progress_not_provider_success():
     assert not engine.is_healthy()
 
 
+def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module():
+    """Flux must restart the exporter with every reliability input mounted."""
+    deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
+    annotations = deployment["spec"]["template"]["metadata"]["annotations"]
+    containers = deployment["spec"]["template"]["spec"]["containers"]
+    exporter = next(item for item in containers if item["name"] == "ai-usage-exporter")
+    environment = {item["name"]: item["value"] for item in exporter["env"]}
+
+    assert annotations["ai.bstein.dev/config-rev"] == "20260823-ai-quota-stability"
+    assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45"
+    assert "ai_usage_polling.py=scripts/ai_usage_polling.py" in (
+        HERMES / "kustomization.yaml"
+    ).read_text()
+
+
 def test_health_endpoint_uses_poller_state_and_ignores_provider_failure():
     mod = load_module()
     collector = mod.Collector()