"""Contract checks for the internal Atlas AI operations dashboard.""" from __future__ import annotations import importlib.util import sys from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py" def load_module(): """Load the dashboard generator without invoking its CLI.""" spec = importlib.util.spec_from_file_location("dashboards_render_atlas_ai", SCRIPT) module = importlib.util.module_from_spec(spec) assert spec and spec.loader sys.modules[spec.name] = module spec.loader.exec_module(module) return module def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): mod = load_module() dashboard = mod.build_ai_dashboard() panels = {panel["title"]: panel for panel in dashboard["panels"]} expressions = "\n".join( target.get("expr", "") for panel in dashboard["panels"] for target in panel.get("targets", []) ) assert dashboard["uid"] == "atlas-ai" assert dashboard["folderUid"] == mod.PRIVATE_FOLDER assert dashboard["refresh"] == "1m" assert "Codex 5h Remaining" in panels assert "Codex 5h Reset In" in panels assert "Codex 7d Remaining" in panels assert "Codex 7d Reset In" in panels assert "Claude 5h Remaining" in panels assert "Claude 5h Reset In" in panels assert "Claude 7d Remaining" in panels assert "Claude 7d Reset In" in panels assert "Codex Routed Tokens (24h)" in panels assert "Codex Routed Tokens (7d)" in panels assert "Claude Routed Tokens (24h)" in panels assert "Claude Routed Tokens (7d)" in panels assert "Provider Access Healthy" in panels assert "Quota Fetch Healthy" in panels assert "Provider Selections (Range)" in panels assert "Claude Quota Auth Expires In" in panels assert "Local Classifier Calls" in panels assert "Hermes Workload CPU (Attribution Proxy)" in panels assert "atlas_ai_quota_remaining_percent" in expressions assert "atlas_ai_quota_reset_timestamp_seconds" in expressions assert "atlas_ai_provider_authenticated" in expressions assert "switchyard_decisions_total" in expressions assert "switchyard_cached_tokens_total" in expressions assert "switchyard_reasoning_tokens_total" in expressions assert 'model=~"(route|worker)/codex/.+"' in expressions assert 'model=~"(route|worker)/claude/.+"' in expressions assert "switchyard_model_call_latency_ms_bucket" in expressions assert "/status" not in expressions assert all( panel.get("description") or panel["type"] == "text" for panel in dashboard["panels"] ) def test_claude_weekly_panel_compares_overall_and_fable_remaining(): """The Claude weekly KPI must show both first-party quota ledgers.""" mod = load_module() panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} targets = panels["Claude 7d Remaining"]["targets"] assert panels["Claude 7d Remaining"]["options"]["textMode"] == "value_and_name" assert [target["legendFormat"] for target in targets] == [ "Overall weekly", "Fable 5 weekly", ] assert 'window="seven_day"' in targets[0]["expr"] assert 'window="seven_day_fable"' in targets[1]["expr"] assert all( "atlas_ai_quota_last_success_timestamp_seconds" in target["expr"] for target in targets ) def test_ai_quota_panels_retain_a_bounded_last_good_snapshot(): """One transient collector miss must not blank every account panel.""" mod = load_module() panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} for title in ( "Codex 5h Remaining", "Codex 5h Reset In", "Codex 7d Remaining", "Codex 7d Reset In", "Claude 5h Remaining", "Claude 5h Reset In", "Claude 7d Remaining", "Claude 7d Reset In", ): expression = panels[title]["targets"][0]["expr"] assert "atlas_ai_quota_last_success_timestamp_seconds" in expression assert "last_over_time" in expression assert "[20m]" in expression assert "1200" in expression assert "atlas_ai_quota_fetch_success" not in expression def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes(): """Recent samples bridge rollouts, while missing telemetry stays explicit.""" mod = load_module() panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} for title in ( "Provider Access Healthy", "Quota Fetch Healthy", "Oldest Quota Sample", ): panel = panels[title] expression = panel["targets"][0]["expr"] assert "last_over_time" in expression assert "[20m]" in expression assert "vector(-1)" in expression mappings = panel["fieldConfig"]["defaults"]["mappings"] assert mappings[0]["options"]["-1"]["text"] == "unavailable" def test_claude_quota_renewal_has_dashboard_and_alert_contracts(): """The finite full-scope grant must warn before Fable telemetry expires.""" mod = load_module() panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]} expiry = panels["Claude Quota Auth Expires In"] assert "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds" in ( expiry["targets"][0]["expr"] ) assert expiry["fieldConfig"]["defaults"]["mappings"][0]["options"]["-1"][ "text" ] == "unavailable" rules = next( yaml.safe_load_all( ( ROOT / "services/monitoring/vmalert-atlas-availability.yaml" ).read_text() ) ) quota_rules = yaml.safe_load(rules["data"]["hermes-ai-quota.yaml"]) alerts = { rule["alert"]: rule for group in quota_rules["groups"] for rule in group["rules"] } assert "HermesClaudeQuotaCredentialExpiring" in alerts assert "HermesClaudeFableQuotaUnavailable" in alerts assert all( rule["labels"]["service"] == "hermes-ai-quota" for rule in alerts.values() ) helm_releases = list( yaml.safe_load_all( (ROOT / "services/monitoring/helmrelease.yaml").read_text() ) ) alertmanager = next( release for release in helm_releases if release.get("kind") == "HelmRelease" and release.get("metadata", {}).get("name") == "alertmanager" ) values = alertmanager["spec"]["values"] assert not values.get("nodeSelector") required = values["affinity"]["nodeAffinity"][ "requiredDuringSchedulingIgnoredDuringExecution" ]["nodeSelectorTerms"][0]["matchExpressions"] assert { "key": "node-role.kubernetes.io/control-plane", "operator": "DoesNotExist", } in required routes = values["config"]["route"]["routes"] assert any( route["receiver"] == "email-platform" and "service = hermes-ai-quota" in route["matchers"] for route in routes ) def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric(): """The first three KPI bands must compare providers without layout gaps.""" mod = load_module() dashboard = mod.build_ai_dashboard() bands = { y: sorted( ( panel for panel in dashboard["panels"] if panel["gridPos"]["y"] == y and panel["gridPos"]["h"] == 4 ), key=lambda panel: panel["gridPos"]["x"], ) for y in (0, 4, 8) } for panels in bands.values(): assert sum(panel["gridPos"]["w"] for panel in panels) == 24 assert panels[0]["gridPos"]["x"] == 0 assert all( left["gridPos"]["x"] + left["gridPos"]["w"] == right["gridPos"]["x"] for left, right in zip(panels, panels[1:]) ) codex_suffixes = [panel["title"].removeprefix("Codex ") for panel in bands[0]] claude_suffixes = [panel["title"].removeprefix("Claude ") for panel in bands[4]] assert codex_suffixes == claude_suffixes assert len(bands[8]) == 8 def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution(): mod = load_module() dashboard = mod.build_ai_dashboard() panels = {panel["title"]: panel for panel in dashboard["panels"]} cost_text = panels["Cost Semantics"]["options"]["content"] proxy_description = panels["Hermes Workload CPU (Attribution Proxy)"]["description"] assert "does not invent API costs" in cost_text assert "subscription" in cost_text assert "proxy" in proxy_description assert "not tenant-slot token labels" in proxy_description