diff --git a/services/bstein-dev-home/kustomization.yaml b/services/bstein-dev-home/kustomization.yaml index 92fcb22b..8dedf9c0 100644 --- a/services/bstein-dev-home/kustomization.yaml +++ b/services/bstein-dev-home/kustomization.yaml @@ -20,9 +20,9 @@ resources: - ingress.yaml images: - name: registry.bstein.dev/bstein/bstein-dev-home-frontend - newTag: 0.1.1-479 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} + newTag: 0.1.1-483 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} - name: registry.bstein.dev/bstein/bstein-dev-home-backend - newTag: 0.1.1-479 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} + newTag: 0.1.1-483 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} configMapGenerator: - name: chat-ai-gateway namespace: bstein-dev-home diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index a06bfa12..8ba2972b 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260823-claude-quota-headers" + ai.bstein.dev/config-rev: "20260823-dual-provider-quota-health" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" @@ -1037,6 +1037,7 @@ spec: - {name: ATLAS_AI_CLAUDE_OAUTH_TOKEN_FILE, value: /claude-oauth-access/token} - {name: ATLAS_AI_CLAUDE_QUERY_TIMEOUT_SECONDS, value: "30"} - {name: ATLAS_AI_CLAUDE_QUOTA_MODEL, value: claude-haiku-4-5-20251001} + - {name: ATLAS_AI_AUTHENTICATION_GRACE_SECONDS, value: "1200"} - {name: ATLAS_AI_PROVIDER_HEALTH_ROOT, value: /provider-health} - {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"} - {name: ATLAS_AI_USAGE_PORT, value: "9010"} diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index bdcf7a4b..9f03de71 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -21,6 +21,10 @@ import ai_usage_polling as polling_engine PROVIDER_HEALTH_ROOT = Path( os.environ.get("ATLAS_AI_PROVIDER_HEALTH_ROOT", "/provider-health") ) +AUTHENTICATION_GRACE_SECONDS = max( + 60, + int(os.environ.get("ATLAS_AI_AUTHENTICATION_GRACE_SECONDS", "1200")), +) CLAUDE_WINDOWS = ( "five_hour", "seven_day", @@ -331,6 +335,10 @@ class Collector: state = self._providers[provider] state.last_attempt = started state.duration = time.monotonic() - monotonic_started + # A successful first-party quota request is direct authentication + # proof. Broker health files may be quiet when no routed model call + # has occurred recently, so they must not demote this live result. + state.authenticated = True state.fetch_success = True state.samples = samples state.last_success = time.time() @@ -342,8 +350,15 @@ class Collector: provider: ProviderState(**vars(state)) for provider, state in self._providers.items() } + now = time.time() for provider, state in states.items(): - state.authenticated = _provider_authenticated(provider) + quota_proves_access = ( + state.last_success > 0 + and 0 <= now - state.last_success <= AUTHENTICATION_GRACE_SECONDS + ) + state.authenticated = ( + _provider_authenticated(provider) or quota_proves_access + ) samples: list[Sample] = [] for provider, state in states.items(): labels = {"provider": provider} diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index 74489179..8b5f61b5 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -263,9 +263,10 @@ def test_manifest_rolls_out_the_bounded_codex_deadline_and_poller_module(): environment = {item["name"]: item["value"] for item in exporter["env"]} assert annotations["ai.bstein.dev/config-rev"] == ( - "20260823-claude-quota-headers" + "20260823-dual-provider-quota-health" ) assert environment["ATLAS_AI_CODEX_QUERY_TIMEOUT_SECONDS"] == "45" + assert environment["ATLAS_AI_AUTHENTICATION_GRACE_SECONDS"] == "1200" assert environment["ATLAS_AI_CLAUDE_OAUTH_TOKEN_FILE"] == ( "/claude-oauth-access/token" ) diff --git a/testing/tests/test_hermes_ai_usage_exporter_coverage.py b/testing/tests/test_hermes_ai_usage_exporter_coverage.py index 881dbe7e..122b8877 100644 --- a/testing/tests/test_hermes_ai_usage_exporter_coverage.py +++ b/testing/tests/test_hermes_ai_usage_exporter_coverage.py @@ -209,6 +209,28 @@ def test_provider_authentication_uses_only_fresh_non_secret_health(tmp_path, mon assert not mod._provider_authenticated("anthropic") +def test_recent_quota_success_keeps_access_healthy_when_broker_snapshot_is_quiet( + monkeypatch, +): + mod = load_module() + collector = mod.Collector() + collector._providers["openai"].last_success = 1_000 + monkeypatch.setattr(mod, "_provider_authenticated", lambda _provider: False) + monkeypatch.setattr(mod.time, "time", lambda: 1_300) + + rendered = collector.render().decode() + + assert 'atlas_ai_provider_authenticated{provider="openai"} 1' in rendered + + monkeypatch.setattr( + mod.time, + "time", + lambda: 1_000 + mod.AUTHENTICATION_GRACE_SECONDS + 1, + ) + rendered = collector.render().decode() + assert 'atlas_ai_provider_authenticated{provider="openai"} 0' in rendered + + def test_collector_refreshes_both_providers_and_contains_unknown_provider( monkeypatch, capsys ): @@ -225,6 +247,8 @@ def test_collector_refreshes_both_providers_and_contains_unknown_provider( collector.refresh_provider("anthropic") assert collector._providers["openai"].fetch_success assert collector._providers["anthropic"].fetch_success + assert collector._providers["openai"].authenticated + assert collector._providers["anthropic"].authenticated assert collector._providers["openai"].last_success > 0 collector.record_failure("openai")