284 lines
16 KiB
Python
284 lines
16 KiB
Python
"""Atlas testing and jobs dashboard rendering contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from test_dashboard_render_support import (
|
|
flatten_panels,
|
|
load_module,
|
|
)
|
|
|
|
|
|
def test_testing_suite_variable_uses_configured_suite_values_only():
|
|
mod = load_module()
|
|
variable = mod.testing_suite_variable()
|
|
configured_matcher = "|".join(mod.PLATFORM_TEST_SUITE_NAMES)
|
|
legacy_names = {"bstein-home", "data-prepper", "titan-iac", "pegasus-health"}
|
|
out_of_scope_names = {"arcanagon", "typhon"}
|
|
|
|
assert variable["allValue"] == configured_matcher
|
|
assert not any(alias in variable["query"] for alias in legacy_names)
|
|
assert not any(alias in variable["allValue"] for alias in legacy_names)
|
|
assert not any(name in variable["query"] for name in out_of_scope_names)
|
|
assert not any(name in variable["allValue"] for name in out_of_scope_names)
|
|
assert [option["value"] for option in variable["options"]] == mod.PLATFORM_TEST_SUITE_NAMES
|
|
assert "lesavka" in variable["allValue"]
|
|
assert any(option["value"] == "lesavka" for option in variable["options"])
|
|
assert not any(
|
|
option["value"] in out_of_scope_names for option in variable["options"]
|
|
)
|
|
|
|
def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
|
|
mod = load_module()
|
|
jobs = mod.build_jobs_dashboard()
|
|
testing = mod.build_testing_dashboard()
|
|
|
|
assert jobs["folderUid"] == mod.PRIVATE_FOLDER
|
|
assert jobs["editable"] is True
|
|
assert testing["uid"] == "atlas-testing"
|
|
assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
|
|
assert testing["editable"] is False
|
|
|
|
def test_jobs_dashboard_separates_current_gate_health_from_reliability():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
variables_by_name = {item["name"]: item for item in dashboard["templating"]["list"]}
|
|
|
|
test_variable = variables_by_name["test"]
|
|
assert test_variable["query"].startswith("query_result(")
|
|
assert "bottomk(75" in test_variable["query"]
|
|
assert "min_over_time" in test_variable["query"]
|
|
assert "platform_quality:test_case_health_rate:percent_1h" in test_variable["query"]
|
|
assert "platform_quality_gate_test_case_result" not in test_variable["query"]
|
|
assert test_variable["regex"] == '/test="([^"]+)"/'
|
|
branch_variable = variables_by_name["branch"]
|
|
assert branch_variable["type"] == "custom"
|
|
assert branch_variable["allValue"] == ".*"
|
|
assert "platform_quality_gate_build_info" not in branch_variable["query"]
|
|
assert "origin/main : origin/main" in branch_variable["query"]
|
|
|
|
assert "Current Gate Health (%)" in panels_by_title
|
|
assert "Suite Freshness (24h)" in panels_by_title
|
|
assert "Latest Gate Health by Suite" in panels_by_title
|
|
assert "Latest Gate Checks Passing by Suite" not in panels_by_title
|
|
assert "CI Run Success by Suite (24h)" in panels_by_title
|
|
assert "CI Run Success by Suite (7d rolling)" in panels_by_title
|
|
assert "Daily Run Volume (Selected Scope)" in panels_by_title
|
|
assert "Coverage History by Suite" in panels_by_title
|
|
assert "Files <=500 LOC History by Suite" in panels_by_title
|
|
assert "Run Reliability History by Suite" not in panels_by_title
|
|
assert "Current Gate Health by Suite" not in panels_by_title
|
|
assert "Coverage & LOC Compliance History" not in panels_by_title
|
|
assert "Run Status Mix (30d)" not in panels_by_title
|
|
assert "Failures by Suite (24h)" not in panels_by_title
|
|
assert "Success Rate by Suite (24h)" not in panels_by_title
|
|
|
|
current_gate_expr = panels_by_title["Latest Gate Health by Suite"]["targets"][0]["expr"]
|
|
assert 'check)' in current_gate_expr
|
|
assert "platform_quality:check_status:present_1h" in current_gate_expr
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_expr
|
|
assert "- 1" in current_gate_expr
|
|
assert '.*_quality_gate_checks_total' not in current_gate_expr
|
|
assert "last_over_time" not in current_gate_expr
|
|
assert 'label_replace' not in current_gate_expr
|
|
assert f'check=~"{mod.PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX}"' in current_gate_expr
|
|
assert f'suite=~"{mod.PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_MATCHER}"' in current_gate_expr
|
|
assert 'check="supply_chain"' in current_gate_expr
|
|
assert f'check=~"{mod.PLATFORM_TEST_STANDARD_CHECK_REGEX}"' not in current_gate_expr
|
|
assert 'status=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
|
|
assert 'status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
|
|
assert "unless on(suite, check)" in current_gate_expr
|
|
assert "tlast_over_time" not in current_gate_expr
|
|
assert panels_by_title["Current Gate Health (%)"]["gridPos"] == {"h": 4, "w": 3, "x": 0, "y": 0}
|
|
current_gate_stat_expr = panels_by_title["Current Gate Health (%)"]["targets"][0]["expr"]
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_stat_expr
|
|
assert "- 1" not in current_gate_stat_expr
|
|
assert panels_by_title["Suite Freshness (24h)"]["gridPos"] == {"h": 4, "w": 3, "x": 15, "y": 0}
|
|
suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
|
|
assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
|
|
assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
|
|
assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
|
|
assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
|
|
assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
|
|
assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
|
|
assert panels_by_title["Files <=500 LOC by Suite (Latest)"]["gridPos"] == {"h": 7, "w": 6, "x": 18, "y": 4}
|
|
|
|
reliability_panel = panels_by_title["CI Run Success by Suite (24h)"]
|
|
reliability_expr = reliability_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_runs:increase_24h" in reliability_expr
|
|
assert "platform_quality_gate_runs_total" not in reliability_expr
|
|
assert "> 0" in reliability_expr
|
|
assert "- 1" in reliability_expr
|
|
assert reliability_expr.startswith("sort(")
|
|
assert reliability_panel["fieldConfig"]["defaults"]["mappings"] == [
|
|
{"type": "value", "options": {"-1": {"text": "no runs"}}}
|
|
]
|
|
|
|
rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
|
|
assert rolling_panel["type"] == "state-timeline"
|
|
assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
|
|
category_panel = panels_by_title["Test Category Health History"]
|
|
assert category_panel["type"] == "state-timeline"
|
|
assert "category" in category_panel["targets"][0]["expr"]
|
|
assert category_panel["targets"][0]["format"] == "time_series"
|
|
assert category_panel["targets"][0]["range"] is True
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in category_panel["targets"][0]["expr"]
|
|
assert 'test!=""' not in category_panel["targets"][0]["expr"]
|
|
assert f'category=~"{mod.PLATFORM_TEST_CATEGORY_REGEX}"' in category_panel["targets"][0]["expr"]
|
|
assert "installer" not in mod.PLATFORM_TEST_CATEGORY_REGEX
|
|
assert "Use the Suite filter" in category_panel["description"]
|
|
assert "skipped tests are healthy" in category_panel["description"]
|
|
|
|
tests_failure_expr = panels_by_title["Tests Failure Rate"]["targets"][0]["expr"]
|
|
assert "platform_quality:check_failed_flag:present_1h" in tests_failure_expr
|
|
assert "platform_quality:check_seen_flag:present_1h" in tests_failure_expr
|
|
assert "platform_quality:check_status:present_1h" not in tests_failure_expr
|
|
|
|
tests_healthy_expr = panels_by_title["Tests Healthy Rate"]["targets"][0]["expr"]
|
|
assert "platform_quality:check_healthy_flag:present_1h" in tests_healthy_expr
|
|
assert "platform_quality:check_seen_flag:present_1h" in tests_healthy_expr
|
|
|
|
problematic_expr = panels_by_title["Problematic Tests Over Time (Top failures)"]["targets"][0]["expr"]
|
|
assert "platform_quality:test_case_status:count_1h" in problematic_expr
|
|
assert "platform_quality_gate_test_case_result" not in problematic_expr
|
|
assert "[24h:1h]" in problematic_expr
|
|
assert ">= 2" in problematic_expr
|
|
assert "@ end()" in problematic_expr
|
|
|
|
coverage_panel = panels_by_title["Coverage History by Suite"]
|
|
loc_panel = panels_by_title["Files <=500 LOC History by Suite"]
|
|
assert coverage_panel["type"] == "state-timeline"
|
|
assert loc_panel["type"] == "state-timeline"
|
|
assert coverage_panel["targets"][0]["expr"] != loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_coverage_percent:latest_1h" in coverage_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_source_files_total:latest_1h" in loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_source_lines_over_500_total:latest_1h" in loc_panel["targets"][0]["expr"]
|
|
assert "tlast_over_time" not in coverage_panel["targets"][0]["expr"]
|
|
assert "tlast_over_time" not in loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality_gate_workspace_line_coverage_percent" not in panels_by_title["Coverage by Suite (Latest, gate 95)"]["targets"][0]["expr"]
|
|
assert "platform_quality:suite_coverage_percent:latest_1h" in panels_by_title["Avg Coverage (%)"]["targets"][0]["expr"]
|
|
assert panels_by_title["Daily Run Volume (Selected Scope)"]["gridPos"] == {"h": 8, "w": 8, "x": 0, "y": 27}
|
|
assert coverage_panel["gridPos"] == {"h": 8, "w": 8, "x": 8, "y": 27}
|
|
assert loc_panel["gridPos"] == {"h": 8, "w": 8, "x": 16, "y": 27}
|
|
|
|
run_volume_panel = panels_by_title["Daily Run Volume (Selected Scope)"]
|
|
assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
|
|
|
|
def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels = flatten_panels(dashboard["panels"])
|
|
bar_gauges = [panel for panel in panels if panel["type"] == "bargauge"]
|
|
|
|
assert bar_gauges
|
|
assert all(panel["options"]["displayMode"] == "basic" for panel in bar_gauges)
|
|
assert all(
|
|
panel["fieldConfig"]["defaults"]["color"]["mode"] == "thresholds"
|
|
for panel in bar_gauges
|
|
)
|
|
|
|
reliability_panel = next(
|
|
panel for panel in panels if panel["title"] == "CI Run Success by Suite (24h)"
|
|
)
|
|
threshold_steps = reliability_panel["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
|
|
|
assert {"color": "dark-yellow", "value": 93} in threshold_steps
|
|
assert {"color": "dark-blue", "value": 100} in threshold_steps
|
|
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels = dashboard["panels"]
|
|
rows = [panel for panel in panels if panel["type"] == "row"]
|
|
visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
|
|
nested_panels_by_title = {
|
|
child["title"]: child
|
|
for row in rows
|
|
for child in row.get("panels", [])
|
|
}
|
|
|
|
assert len(panels) == 18
|
|
assert len(visible_query_panels) == 12
|
|
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
|
|
assert all(
|
|
panel["title"] != "Coverage Gap to 95% by Suite"
|
|
for panel in visible_query_panels
|
|
)
|
|
assert [row["title"] for row in rows] == [
|
|
"CI Runs And Test Result History",
|
|
"Check Failure Rates By Suite",
|
|
"Check Healthy Rates By Suite",
|
|
"Test Drilldowns And Problem Tests",
|
|
"Telemetry Completeness And Branches",
|
|
"SonarQube Project Health",
|
|
]
|
|
assert all(row["collapsed"] for row in rows)
|
|
|
|
assert "Coverage Failure Rate" in nested_panels_by_title
|
|
assert "Supply Chain Healthy Rate" in nested_panels_by_title
|
|
assert "Test Category Health History" in nested_panels_by_title
|
|
assert "Selected Test Pass Rate History" in nested_panels_by_title
|
|
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
|
|
assert "SonarQube API Up" in nested_panels_by_title
|
|
|
|
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
|
|
assert failure_rate_panel["type"] == "state-timeline"
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue"
|
|
assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "0 *" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
|
|
|
|
pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
|
|
assert pass_rate_panel["type"] == "state-timeline"
|
|
assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"]
|
|
|
|
pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
|
|
assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
assert all(
|
|
"platform_quality:test_case_status:count_1h" in target["expr"]
|
|
for target in pass_fail_panel["targets"]
|
|
)
|
|
|
|
problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"]
|
|
assert problematic_panel["type"] == "state-timeline"
|
|
assert problematic_panel["gridPos"]["w"] == 24
|
|
assert 'test!=""' in problematic_panel["targets"][0]["expr"]
|
|
assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
|
|
assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
|
|
{"color": "dark-blue", "value": None},
|
|
{"color": "dark-green", "value": 2},
|
|
{"color": "dark-yellow", "value": 3},
|
|
{"color": "dark-orange", "value": 5},
|
|
{"color": "dark-red", "value": 8},
|
|
]
|
|
assert "rolling 24h failure count" in problematic_panel["description"]
|
|
assert "at least two recent failures" in problematic_panel["description"]
|
|
|
|
sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
|
|
sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
|
|
assert sonar_mix_panel["gridPos"]["w"] == 4
|
|
assert sonar_health_panel["gridPos"]["w"] == 8
|
|
assert sonar_health_panel["type"] == "state-timeline"
|
|
assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
|
|
assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
|
|
|
|
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
|
|
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
|
|
assert branch_panel["gridPos"]["x"] == 12
|
|
assert recent_branch_panel["gridPos"]["x"] == 18
|
|
assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
|
|
assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
|
|
assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
|
|
assert "> bool 0" in branch_panel["targets"][0]["expr"]
|
|
assert branch_panel["targets"][0]["expr"].startswith("sort(")
|