03b10ff8b1
The distributed worker pool stacks on exactly two open pull requests, in this order: PR 14 supplies the broker-only SCM boundary the mediators route through and the hermes-scm-boundary-v2 ConfigMap they mount, and PR 15 supplies the cli_lane_* decomposition -- including canonical_run_id and the eligibility predicate on claim_ready -- that the coordinator depends on. Neither can be dropped without breaking a fixed P0 boundary, so both are carried here as prerequisites and this branch must not merge before them. PR 16 (agent image release lane) and PR 19 (full-handoff acceptance harness) are NOT prerequisites and are deliberately absent, so reviewing this branch no longer means approving them. PR 14 and PR 15 conflict with each other in nine paths. Each is resolved to the resolution already reviewed on this branch at 4d4cf1bd.
282 lines
16 KiB
Python
282 lines
16 KiB
Python
"""Atlas testing and jobs dashboard rendering contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from test_dashboard_render_support import (
|
|
flatten_panels,
|
|
load_module,
|
|
)
|
|
|
|
|
|
def test_testing_suite_variable_uses_configured_suite_values_only():
|
|
mod = load_module()
|
|
variable = mod.testing_suite_variable()
|
|
configured_matcher = "|".join(mod.PLATFORM_TEST_SUITE_NAMES)
|
|
legacy_names = {"bstein-home", "data-prepper", "titan-iac", "pegasus-health"}
|
|
out_of_scope_names = {"arcanagon", "typhon"}
|
|
|
|
assert variable["allValue"] == configured_matcher
|
|
assert not any(alias in variable["query"] for alias in legacy_names)
|
|
assert not any(alias in variable["allValue"] for alias in legacy_names)
|
|
assert not any(name in variable["query"] for name in out_of_scope_names)
|
|
assert not any(name in variable["allValue"] for name in out_of_scope_names)
|
|
assert [option["value"] for option in variable["options"]] == mod.PLATFORM_TEST_SUITE_NAMES
|
|
assert "lesavka" in variable["allValue"]
|
|
assert any(option["value"] == "lesavka" for option in variable["options"])
|
|
assert not any(
|
|
option["value"] in out_of_scope_names for option in variable["options"]
|
|
)
|
|
|
|
def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
|
|
mod = load_module()
|
|
jobs = mod.build_jobs_dashboard()
|
|
testing = mod.build_testing_dashboard()
|
|
|
|
assert jobs["folderUid"] == mod.PRIVATE_FOLDER
|
|
assert jobs["editable"] is True
|
|
assert testing["uid"] == "atlas-testing"
|
|
assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
|
|
assert testing["editable"] is False
|
|
|
|
def test_jobs_dashboard_separates_current_gate_health_from_reliability():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
variables_by_name = {item["name"]: item for item in dashboard["templating"]["list"]}
|
|
|
|
test_variable = variables_by_name["test"]
|
|
assert test_variable["query"].startswith("query_result(")
|
|
assert "platform_quality:test_case_health_rate:percent_1h" in test_variable["query"]
|
|
assert "platform_quality_gate_test_case_result" not in test_variable["query"]
|
|
assert test_variable["regex"] == '/test="([^"]+)"/'
|
|
branch_variable = variables_by_name["branch"]
|
|
assert branch_variable["type"] == "custom"
|
|
assert branch_variable["allValue"] == ".*"
|
|
assert "platform_quality_gate_build_info" not in branch_variable["query"]
|
|
assert "origin/main : origin/main" in branch_variable["query"]
|
|
|
|
assert "Current Gate Health (%)" in panels_by_title
|
|
assert "Suite Freshness (24h)" in panels_by_title
|
|
assert "Latest Gate Health by Suite" in panels_by_title
|
|
assert "Latest Gate Checks Passing by Suite" not in panels_by_title
|
|
assert "CI Run Success by Suite (24h)" in panels_by_title
|
|
assert "CI Run Success by Suite (7d rolling)" in panels_by_title
|
|
assert "Daily Run Volume (Selected Scope)" in panels_by_title
|
|
assert "Coverage History by Suite" in panels_by_title
|
|
assert "Files <=500 LOC History by Suite" in panels_by_title
|
|
assert "Run Reliability History by Suite" not in panels_by_title
|
|
assert "Current Gate Health by Suite" not in panels_by_title
|
|
assert "Coverage & LOC Compliance History" not in panels_by_title
|
|
assert "Run Status Mix (30d)" not in panels_by_title
|
|
assert "Failures by Suite (24h)" not in panels_by_title
|
|
assert "Success Rate by Suite (24h)" not in panels_by_title
|
|
|
|
current_gate_expr = panels_by_title["Latest Gate Health by Suite"]["targets"][0]["expr"]
|
|
assert 'check)' in current_gate_expr
|
|
assert "platform_quality:check_status:present_1h" in current_gate_expr
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_expr
|
|
assert "- 1" in current_gate_expr
|
|
assert '.*_quality_gate_checks_total' not in current_gate_expr
|
|
assert "last_over_time" not in current_gate_expr
|
|
assert 'label_replace' not in current_gate_expr
|
|
assert f'check=~"{mod.PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX}"' in current_gate_expr
|
|
assert f'suite=~"{mod.PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_MATCHER}"' in current_gate_expr
|
|
assert 'check="supply_chain"' in current_gate_expr
|
|
assert f'check=~"{mod.PLATFORM_TEST_STANDARD_CHECK_REGEX}"' not in current_gate_expr
|
|
assert 'status=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
|
|
assert 'status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
|
|
assert "unless on(suite, check)" in current_gate_expr
|
|
assert "tlast_over_time" not in current_gate_expr
|
|
assert panels_by_title["Current Gate Health (%)"]["gridPos"] == {"h": 4, "w": 3, "x": 0, "y": 0}
|
|
current_gate_stat_expr = panels_by_title["Current Gate Health (%)"]["targets"][0]["expr"]
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_stat_expr
|
|
assert "- 1" not in current_gate_stat_expr
|
|
assert panels_by_title["Suite Freshness (24h)"]["gridPos"] == {"h": 4, "w": 3, "x": 15, "y": 0}
|
|
suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
|
|
assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
|
|
assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
|
|
assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
|
|
assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
|
|
assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
|
|
assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
|
|
assert panels_by_title["Files <=500 LOC by Suite (Latest)"]["gridPos"] == {"h": 7, "w": 6, "x": 18, "y": 4}
|
|
|
|
reliability_panel = panels_by_title["CI Run Success by Suite (24h)"]
|
|
reliability_expr = reliability_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_runs:increase_24h" in reliability_expr
|
|
assert "platform_quality_gate_runs_total" not in reliability_expr
|
|
assert "> 0" in reliability_expr
|
|
assert "- 1" in reliability_expr
|
|
assert reliability_expr.startswith("sort(")
|
|
assert reliability_panel["fieldConfig"]["defaults"]["mappings"] == [
|
|
{"type": "value", "options": {"-1": {"text": "no runs"}}}
|
|
]
|
|
|
|
rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
|
|
assert rolling_panel["type"] == "state-timeline"
|
|
assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
|
|
category_panel = panels_by_title["Test Category Health History"]
|
|
assert category_panel["type"] == "state-timeline"
|
|
assert "category" in category_panel["targets"][0]["expr"]
|
|
assert category_panel["targets"][0]["format"] == "time_series"
|
|
assert category_panel["targets"][0]["range"] is True
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in category_panel["targets"][0]["expr"]
|
|
assert 'test!=""' not in category_panel["targets"][0]["expr"]
|
|
assert f'category=~"{mod.PLATFORM_TEST_CATEGORY_REGEX}"' in category_panel["targets"][0]["expr"]
|
|
assert "installer" not in mod.PLATFORM_TEST_CATEGORY_REGEX
|
|
assert "Use the Suite filter" in category_panel["description"]
|
|
assert "skipped tests are healthy" in category_panel["description"]
|
|
|
|
tests_failure_expr = panels_by_title["Tests Failure Rate"]["targets"][0]["expr"]
|
|
assert "platform_quality:check_failed_flag:present_1h" in tests_failure_expr
|
|
assert "platform_quality:check_seen_flag:present_1h" in tests_failure_expr
|
|
assert "platform_quality:check_status:present_1h" not in tests_failure_expr
|
|
|
|
tests_healthy_expr = panels_by_title["Tests Healthy Rate"]["targets"][0]["expr"]
|
|
assert "platform_quality:check_healthy_flag:present_1h" in tests_healthy_expr
|
|
assert "platform_quality:check_seen_flag:present_1h" in tests_healthy_expr
|
|
|
|
problematic_expr = panels_by_title["Problematic Tests Over Time (Top failures)"]["targets"][0]["expr"]
|
|
assert "platform_quality:test_case_status:count_1h" in problematic_expr
|
|
assert "platform_quality_gate_test_case_result" not in problematic_expr
|
|
assert "[24h:1h]" in problematic_expr
|
|
assert ">= 2" in problematic_expr
|
|
assert "@ end()" in problematic_expr
|
|
|
|
coverage_panel = panels_by_title["Coverage History by Suite"]
|
|
loc_panel = panels_by_title["Files <=500 LOC History by Suite"]
|
|
assert coverage_panel["type"] == "state-timeline"
|
|
assert loc_panel["type"] == "state-timeline"
|
|
assert coverage_panel["targets"][0]["expr"] != loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_coverage_percent:latest_1h" in coverage_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_source_files_total:latest_1h" in loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality:suite_source_lines_over_500_total:latest_1h" in loc_panel["targets"][0]["expr"]
|
|
assert "tlast_over_time" not in coverage_panel["targets"][0]["expr"]
|
|
assert "tlast_over_time" not in loc_panel["targets"][0]["expr"]
|
|
assert "platform_quality_gate_workspace_line_coverage_percent" not in panels_by_title["Coverage by Suite (Latest, gate 95)"]["targets"][0]["expr"]
|
|
assert "platform_quality:suite_coverage_percent:latest_1h" in panels_by_title["Avg Coverage (%)"]["targets"][0]["expr"]
|
|
assert panels_by_title["Daily Run Volume (Selected Scope)"]["gridPos"] == {"h": 8, "w": 8, "x": 0, "y": 27}
|
|
assert coverage_panel["gridPos"] == {"h": 8, "w": 8, "x": 8, "y": 27}
|
|
assert loc_panel["gridPos"] == {"h": 8, "w": 8, "x": 16, "y": 27}
|
|
|
|
run_volume_panel = panels_by_title["Daily Run Volume (Selected Scope)"]
|
|
assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
|
|
|
|
def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels = flatten_panels(dashboard["panels"])
|
|
bar_gauges = [panel for panel in panels if panel["type"] == "bargauge"]
|
|
|
|
assert bar_gauges
|
|
assert all(panel["options"]["displayMode"] == "basic" for panel in bar_gauges)
|
|
assert all(
|
|
panel["fieldConfig"]["defaults"]["color"]["mode"] == "thresholds"
|
|
for panel in bar_gauges
|
|
)
|
|
|
|
reliability_panel = next(
|
|
panel for panel in panels if panel["title"] == "CI Run Success by Suite (24h)"
|
|
)
|
|
threshold_steps = reliability_panel["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
|
|
|
assert {"color": "dark-yellow", "value": 93} in threshold_steps
|
|
assert {"color": "dark-blue", "value": 100} in threshold_steps
|
|
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
|
|
mod = load_module()
|
|
dashboard = mod.build_jobs_dashboard()
|
|
panels = dashboard["panels"]
|
|
rows = [panel for panel in panels if panel["type"] == "row"]
|
|
visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
|
|
nested_panels_by_title = {
|
|
child["title"]: child
|
|
for row in rows
|
|
for child in row.get("panels", [])
|
|
}
|
|
|
|
assert len(panels) == 18
|
|
assert len(visible_query_panels) == 12
|
|
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
|
|
assert all(
|
|
panel["title"] != "Coverage Gap to 95% by Suite"
|
|
for panel in visible_query_panels
|
|
)
|
|
assert [row["title"] for row in rows] == [
|
|
"CI Runs And Test Result History",
|
|
"Check Failure Rates By Suite",
|
|
"Check Healthy Rates By Suite",
|
|
"Test Drilldowns And Problem Tests",
|
|
"Telemetry Completeness And Branches",
|
|
"SonarQube Project Health",
|
|
]
|
|
assert all(row["collapsed"] for row in rows)
|
|
|
|
assert "Coverage Failure Rate" in nested_panels_by_title
|
|
assert "Supply Chain Healthy Rate" in nested_panels_by_title
|
|
assert "Test Category Health History" in nested_panels_by_title
|
|
assert "Selected Test Pass Rate History" in nested_panels_by_title
|
|
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
|
|
assert "SonarQube API Up" in nested_panels_by_title
|
|
|
|
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
|
|
assert failure_rate_panel["type"] == "state-timeline"
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
|
|
assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue"
|
|
assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
|
|
assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"]
|
|
assert "0 *" in failure_rate_panel["targets"][0]["expr"]
|
|
assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
|
|
|
|
pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
|
|
assert pass_rate_panel["type"] == "state-timeline"
|
|
assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"]
|
|
assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"]
|
|
|
|
pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
|
|
assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
assert all(
|
|
"platform_quality:test_case_status:count_1h" in target["expr"]
|
|
for target in pass_fail_panel["targets"]
|
|
)
|
|
|
|
problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"]
|
|
assert problematic_panel["type"] == "state-timeline"
|
|
assert problematic_panel["gridPos"]["w"] == 24
|
|
assert 'test!=""' in problematic_panel["targets"][0]["expr"]
|
|
assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
|
|
assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
|
|
{"color": "dark-blue", "value": None},
|
|
{"color": "dark-green", "value": 2},
|
|
{"color": "dark-yellow", "value": 3},
|
|
{"color": "dark-orange", "value": 5},
|
|
{"color": "dark-red", "value": 8},
|
|
]
|
|
assert "rolling 24h failure count" in problematic_panel["description"]
|
|
assert "at least two recent failures" in problematic_panel["description"]
|
|
|
|
sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
|
|
sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
|
|
assert sonar_mix_panel["gridPos"]["w"] == 4
|
|
assert sonar_health_panel["gridPos"]["w"] == 8
|
|
assert sonar_health_panel["type"] == "state-timeline"
|
|
assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
|
|
assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
|
|
|
|
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
|
|
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
|
|
assert branch_panel["gridPos"]["x"] == 12
|
|
assert recent_branch_panel["gridPos"]["x"] == 18
|
|
assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
|
|
assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
|
|
assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
|
|
assert "> bool 0" in branch_panel["targets"][0]["expr"]
|
|
assert branch_panel["targets"][0]["expr"].startswith("sort(")
|