152 lines
5.7 KiB
Python
152 lines
5.7 KiB
Python
"""Contract checks for the internal Atlas AI operations dashboard."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py"
|
|
|
|
|
|
def load_module():
|
|
"""Load the dashboard generator without invoking its CLI."""
|
|
spec = importlib.util.spec_from_file_location("dashboards_render_atlas_ai", SCRIPT)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
panels = {panel["title"]: panel for panel in dashboard["panels"]}
|
|
expressions = "\n".join(
|
|
target.get("expr", "")
|
|
for panel in dashboard["panels"]
|
|
for target in panel.get("targets", [])
|
|
)
|
|
|
|
assert dashboard["uid"] == "atlas-ai"
|
|
assert dashboard["folderUid"] == mod.PRIVATE_FOLDER
|
|
assert dashboard["refresh"] == "1m"
|
|
assert "Codex 5h Remaining" in panels
|
|
assert "Codex 5h Reset In" in panels
|
|
assert "Codex 7d Remaining" in panels
|
|
assert "Codex 7d Reset In" in panels
|
|
assert "Claude 5h Remaining" in panels
|
|
assert "Claude 5h Reset In" in panels
|
|
assert "Claude 7d Remaining" in panels
|
|
assert "Claude 7d Reset In" in panels
|
|
assert "Codex Routed Tokens (24h)" in panels
|
|
assert "Codex Routed Tokens (7d)" in panels
|
|
assert "Claude Routed Tokens (24h)" in panels
|
|
assert "Claude Routed Tokens (7d)" in panels
|
|
assert "Provider Access Healthy" in panels
|
|
assert "Quota Fetch Healthy" in panels
|
|
assert "Provider Selections (Range)" in panels
|
|
assert "Local Classifier Calls" in panels
|
|
assert "Hermes Workload CPU (Attribution Proxy)" in panels
|
|
assert "atlas_ai_quota_remaining_percent" in expressions
|
|
assert "atlas_ai_quota_reset_timestamp_seconds" in expressions
|
|
assert "atlas_ai_provider_authenticated" in expressions
|
|
assert "switchyard_decisions_total" in expressions
|
|
assert "switchyard_cached_tokens_total" in expressions
|
|
assert "switchyard_reasoning_tokens_total" in expressions
|
|
assert 'model=~"(route|worker)/codex/.+"' in expressions
|
|
assert 'model=~"(route|worker)/claude/.+"' in expressions
|
|
assert "switchyard_model_call_latency_ms_bucket" in expressions
|
|
assert "/status" not in expressions
|
|
assert all(
|
|
panel.get("description") or panel["type"] == "text"
|
|
for panel in dashboard["panels"]
|
|
)
|
|
|
|
|
|
def test_claude_weekly_panel_compares_overall_and_fable_remaining():
|
|
"""The Claude weekly KPI must show both first-party quota ledgers."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
targets = panels["Claude 7d Remaining"]["targets"]
|
|
|
|
assert panels["Claude 7d Remaining"]["options"]["textMode"] == "value_and_name"
|
|
assert [target["legendFormat"] for target in targets] == [
|
|
"Overall weekly",
|
|
"Fable 5 weekly",
|
|
]
|
|
assert 'window="seven_day"' in targets[0]["expr"]
|
|
assert 'window="seven_day_fable"' in targets[1]["expr"]
|
|
assert all(
|
|
"atlas_ai_quota_last_success_timestamp_seconds" in target["expr"]
|
|
for target in targets
|
|
)
|
|
|
|
|
|
def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
|
|
"""One transient collector miss must not blank every account panel."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
|
|
for title in (
|
|
"Codex 5h Remaining",
|
|
"Codex 5h Reset In",
|
|
"Codex 7d Remaining",
|
|
"Codex 7d Reset In",
|
|
"Claude 5h Remaining",
|
|
"Claude 5h Reset In",
|
|
"Claude 7d Remaining",
|
|
"Claude 7d Reset In",
|
|
):
|
|
expression = panels[title]["targets"][0]["expr"]
|
|
assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
|
|
assert "1200" in expression
|
|
assert "atlas_ai_quota_fetch_success" not in expression
|
|
|
|
|
|
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
|
|
"""The first three KPI bands must compare providers without layout gaps."""
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
bands = {
|
|
y: sorted(
|
|
(
|
|
panel
|
|
for panel in dashboard["panels"]
|
|
if panel["gridPos"]["y"] == y and panel["gridPos"]["h"] == 4
|
|
),
|
|
key=lambda panel: panel["gridPos"]["x"],
|
|
)
|
|
for y in (0, 4, 8)
|
|
}
|
|
|
|
for panels in bands.values():
|
|
assert sum(panel["gridPos"]["w"] for panel in panels) == 24
|
|
assert panels[0]["gridPos"]["x"] == 0
|
|
assert all(
|
|
left["gridPos"]["x"] + left["gridPos"]["w"]
|
|
== right["gridPos"]["x"]
|
|
for left, right in zip(panels, panels[1:])
|
|
)
|
|
|
|
codex_suffixes = [panel["title"].removeprefix("Codex ") for panel in bands[0]]
|
|
claude_suffixes = [panel["title"].removeprefix("Claude ") for panel in bands[4]]
|
|
assert codex_suffixes == claude_suffixes
|
|
assert len(bands[8]) == 8
|
|
|
|
|
|
def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution():
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
panels = {panel["title"]: panel for panel in dashboard["panels"]}
|
|
|
|
cost_text = panels["Cost Semantics"]["options"]["content"]
|
|
proxy_description = panels["Hermes Workload CPU (Attribution Proxy)"]["description"]
|
|
assert "does not invent API costs" in cost_text
|
|
assert "subscription" in cost_text
|
|
assert "proxy" in proxy_description
|
|
assert "not tenant-slot token labels" in proxy_description
|