237 lines
8.5 KiB
Python
237 lines
8.5 KiB
Python
"""Contract checks for the internal Atlas AI operations dashboard."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts/render/dashboards_render_atlas.py"
|
|
|
|
|
|
def load_module():
|
|
"""Load the dashboard generator without invoking its CLI."""
|
|
spec = importlib.util.spec_from_file_location("dashboards_render_atlas_ai", SCRIPT)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
panels = {panel["title"]: panel for panel in dashboard["panels"]}
|
|
expressions = "\n".join(
|
|
target.get("expr", "")
|
|
for panel in dashboard["panels"]
|
|
for target in panel.get("targets", [])
|
|
)
|
|
|
|
assert dashboard["uid"] == "atlas-ai"
|
|
assert dashboard["folderUid"] == mod.PRIVATE_FOLDER
|
|
assert dashboard["refresh"] == "1m"
|
|
assert "Codex 5h Remaining" in panels
|
|
assert "Codex 5h Reset In" in panels
|
|
assert "Codex 7d Remaining" in panels
|
|
assert "Codex 7d Reset In" in panels
|
|
assert "Claude 5h Remaining" in panels
|
|
assert "Claude 5h Reset In" in panels
|
|
assert "Claude 7d Remaining" in panels
|
|
assert "Claude 7d Reset In" in panels
|
|
assert "Codex Routed Tokens (24h)" in panels
|
|
assert "Codex Routed Tokens (7d)" in panels
|
|
assert "Claude Routed Tokens (24h)" in panels
|
|
assert "Claude Routed Tokens (7d)" in panels
|
|
assert "Provider Access Healthy" in panels
|
|
assert "Quota Fetch Healthy" in panels
|
|
assert "Provider Selections (Range)" in panels
|
|
assert "Claude Quota Auth Expires In" in panels
|
|
assert "Local Classifier Calls" in panels
|
|
assert "Hermes Workload CPU (Attribution Proxy)" in panels
|
|
assert "atlas_ai_quota_remaining_percent" in expressions
|
|
assert "atlas_ai_quota_reset_timestamp_seconds" in expressions
|
|
assert "atlas_ai_provider_authenticated" in expressions
|
|
assert "switchyard_decisions_total" in expressions
|
|
assert "switchyard_cached_tokens_total" in expressions
|
|
assert "switchyard_reasoning_tokens_total" in expressions
|
|
assert 'model=~"(route|worker)/codex/.+"' in expressions
|
|
assert 'model=~"(route|worker)/claude/.+"' in expressions
|
|
assert "switchyard_model_call_latency_ms_bucket" in expressions
|
|
assert "/status" not in expressions
|
|
assert all(
|
|
panel.get("description") or panel["type"] == "text"
|
|
for panel in dashboard["panels"]
|
|
)
|
|
|
|
|
|
def test_claude_weekly_panel_compares_overall_and_fable_remaining():
|
|
"""The Claude weekly KPI must show both first-party quota ledgers."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
targets = panels["Claude 7d Remaining"]["targets"]
|
|
|
|
assert panels["Claude 7d Remaining"]["options"]["textMode"] == "value_and_name"
|
|
assert [target["legendFormat"] for target in targets] == [
|
|
"Overall weekly",
|
|
"Fable 5 weekly",
|
|
]
|
|
assert 'window="seven_day"' in targets[0]["expr"]
|
|
assert 'window="seven_day_fable"' in targets[1]["expr"]
|
|
assert all(
|
|
"atlas_ai_quota_last_success_timestamp_seconds" in target["expr"]
|
|
for target in targets
|
|
)
|
|
|
|
|
|
def test_ai_quota_panels_retain_a_bounded_last_good_snapshot():
|
|
"""One transient collector miss must not blank every account panel."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
|
|
for title in (
|
|
"Codex 5h Remaining",
|
|
"Codex 5h Reset In",
|
|
"Codex 7d Remaining",
|
|
"Codex 7d Reset In",
|
|
"Claude 5h Remaining",
|
|
"Claude 5h Reset In",
|
|
"Claude 7d Remaining",
|
|
"Claude 7d Reset In",
|
|
):
|
|
expression = panels[title]["targets"][0]["expr"]
|
|
assert "atlas_ai_quota_last_success_timestamp_seconds" in expression
|
|
assert "last_over_time" in expression
|
|
assert "[20m]" in expression
|
|
assert "1200" in expression
|
|
assert "atlas_ai_quota_fetch_success" not in expression
|
|
|
|
|
|
def test_ai_quota_health_panels_distinguish_rollouts_from_real_zeroes():
|
|
"""Recent samples bridge rollouts, while missing telemetry stays explicit."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
|
|
for title in (
|
|
"Provider Access Healthy",
|
|
"Quota Fetch Healthy",
|
|
"Oldest Quota Sample",
|
|
):
|
|
panel = panels[title]
|
|
expression = panel["targets"][0]["expr"]
|
|
assert "last_over_time" in expression
|
|
assert "[20m]" in expression
|
|
assert "vector(-1)" in expression
|
|
mappings = panel["fieldConfig"]["defaults"]["mappings"]
|
|
assert mappings[0]["options"]["-1"]["text"] == "unavailable"
|
|
|
|
|
|
def test_claude_quota_renewal_has_dashboard_and_alert_contracts():
|
|
"""The finite full-scope grant must warn before Fable telemetry expires."""
|
|
mod = load_module()
|
|
panels = {panel["title"]: panel for panel in mod.build_ai_dashboard()["panels"]}
|
|
expiry = panels["Claude Quota Auth Expires In"]
|
|
|
|
assert "atlas_ai_quota_credential_refresh_expiry_timestamp_seconds" in (
|
|
expiry["targets"][0]["expr"]
|
|
)
|
|
assert expiry["fieldConfig"]["defaults"]["mappings"][0]["options"]["-1"][
|
|
"text"
|
|
] == "unavailable"
|
|
|
|
rules = next(
|
|
yaml.safe_load_all(
|
|
(
|
|
ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
).read_text()
|
|
)
|
|
)
|
|
quota_rules = yaml.safe_load(rules["data"]["hermes-ai-quota.yaml"])
|
|
alerts = {
|
|
rule["alert"]: rule
|
|
for group in quota_rules["groups"]
|
|
for rule in group["rules"]
|
|
}
|
|
assert "HermesClaudeQuotaCredentialExpiring" in alerts
|
|
assert "HermesClaudeFableQuotaUnavailable" in alerts
|
|
assert all(
|
|
rule["labels"]["service"] == "hermes-ai-quota"
|
|
for rule in alerts.values()
|
|
)
|
|
|
|
helm_releases = list(
|
|
yaml.safe_load_all(
|
|
(ROOT / "services/monitoring/helmrelease.yaml").read_text()
|
|
)
|
|
)
|
|
alertmanager = next(
|
|
release
|
|
for release in helm_releases
|
|
if release.get("kind") == "HelmRelease"
|
|
and release.get("metadata", {}).get("name") == "alertmanager"
|
|
)
|
|
values = alertmanager["spec"]["values"]
|
|
assert not values.get("nodeSelector")
|
|
required = values["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"][0]["matchExpressions"]
|
|
assert {
|
|
"key": "node-role.kubernetes.io/control-plane",
|
|
"operator": "DoesNotExist",
|
|
} in required
|
|
routes = values["config"]["route"]["routes"]
|
|
assert any(
|
|
route["receiver"] == "email-platform"
|
|
and "service = hermes-ai-quota" in route["matchers"]
|
|
for route in routes
|
|
)
|
|
|
|
|
|
def test_ai_dashboard_top_bands_are_full_width_and_provider_symmetric():
|
|
"""The first three KPI bands must compare providers without layout gaps."""
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
bands = {
|
|
y: sorted(
|
|
(
|
|
panel
|
|
for panel in dashboard["panels"]
|
|
if panel["gridPos"]["y"] == y and panel["gridPos"]["h"] == 4
|
|
),
|
|
key=lambda panel: panel["gridPos"]["x"],
|
|
)
|
|
for y in (0, 4, 8)
|
|
}
|
|
|
|
for panels in bands.values():
|
|
assert sum(panel["gridPos"]["w"] for panel in panels) == 24
|
|
assert panels[0]["gridPos"]["x"] == 0
|
|
assert all(
|
|
left["gridPos"]["x"] + left["gridPos"]["w"]
|
|
== right["gridPos"]["x"]
|
|
for left, right in zip(panels, panels[1:])
|
|
)
|
|
|
|
codex_suffixes = [panel["title"].removeprefix("Codex ") for panel in bands[0]]
|
|
claude_suffixes = [panel["title"].removeprefix("Claude ") for panel in bands[4]]
|
|
assert codex_suffixes == claude_suffixes
|
|
assert len(bands[8]) == 8
|
|
|
|
|
|
def test_ai_dashboard_cost_and_instance_panels_do_not_claim_false_attribution():
|
|
mod = load_module()
|
|
dashboard = mod.build_ai_dashboard()
|
|
panels = {panel["title"]: panel for panel in dashboard["panels"]}
|
|
|
|
cost_text = panels["Cost Semantics"]["options"]["content"]
|
|
proxy_description = panels["Hermes Workload CPU (Attribution Proxy)"]["description"]
|
|
assert "does not invent API costs" in cost_text
|
|
assert "subscription" in cost_text
|
|
assert "proxy" in proxy_description
|
|
assert "not tenant-slot token labels" in proxy_description
|