monitoring(testing): ignore stale replaced check states

scripts/dashboards_render_atlas.py

@@ -635,32 +635,46 @@ def platform_check_status_expr(
     branch_matcher: str = 'branch!=""',
     check_matcher: str = 'check!=""',
     status_matcher: str = 'status!=""',
-    window: str = PLATFORM_TEST_FRESH_WINDOW,
+    window: str | None = PLATFORM_TEST_FRESH_WINDOW,
 ) -> str:
     """Return recent check gauges normalized to a status label."""
     result_matcher = status_matcher.replace("status", "result", 1)
+    status_guards = [status_matcher]
+    result_guards = [result_matcher]
+    if status_matcher.startswith("status!") and status_matcher != 'status!=""':
+        status_guards.insert(0, 'status!=""')
+    if result_matcher.startswith("result!") and result_matcher != 'result!=""':
+        result_guards.insert(0, 'result!=""')
+    status_selector = ",".join(status_guards)
+    result_selector = ",".join(result_guards)
     base = (
         f'__name__=~".*_quality_gate_checks_total",{suite_matcher},'
         f'{PLATFORM_TEST_EXPORT_FILTER},{check_matcher}'
     )
+    def maybe_recent(selector: str) -> str:
+        metric = f"{{{selector}}}"
+        return f"last_over_time({metric}[{window}])" if window else metric
+
+    build_info_metric = f'platform_quality_gate_build_info{{{suite_matcher},{PLATFORM_TEST_EXPORT_FILTER},{branch_matcher}}}'
     build_info = (
-        f'max by (suite, branch) (last_over_time(platform_quality_gate_build_info'
-        f'{{{suite_matcher},{PLATFORM_TEST_EXPORT_FILTER},{branch_matcher}}}[{window}]))'
+        f"max by (suite, branch) ("
+        f"{f'last_over_time({build_info_metric}[{window}])' if window else build_info_metric}"
+        f")"
     )
     with_status = (
-        f'last_over_time({{{base},{branch_matcher},{status_matcher}}}[{window}])'
+        maybe_recent(f"{base},{branch_matcher},{status_selector}")
     )
     with_result = (
-        f'label_replace(last_over_time({{{base},{branch_matcher},{result_matcher},status=""}}'
-        f'[{window}]), "status", "$1", "result", "(.*)")'
+        f'label_replace({maybe_recent(f'{base},{branch_matcher},{result_selector},status=""')}, '
+        f'"status", "$1", "result", "(.*)")'
     )
     inferred_status = (
-        f'(last_over_time({{{base},branch="",{status_matcher}}}[{window}]) '
+        f'({maybe_recent(f"{base},branch=\"\",{status_selector}")} '
         f'* on (suite) group_left(branch) ({build_info}))'
     )
     inferred_result = (
-        f'(label_replace(last_over_time({{{base},branch="",{result_matcher},status=""}}'
-        f'[{window}]), "status", "$1", "result", "(.*)") '
+        f'(label_replace({maybe_recent(f"{base},branch=\"\",{result_selector},status=\"\"")}, '
+        f'"status", "$1", "result", "(.*)") '
         f'* on (suite) group_left(branch) ({build_info}))'
     )
     return (
@@ -728,12 +742,14 @@ PLATFORM_TEST_PRIMARY_BRANCH_MATCHER = 'branch=~"main|master|origin/main|origin/
 PLATFORM_TEST_CHECK_ROLLUP_SELECTOR = platform_check_status_expr(
     PLATFORM_TEST_CHECK_ROLLUP_MATCHERS,
     branch_matcher=PLATFORM_TEST_PRIMARY_BRANCH_MATCHER,
+    window=None,
 )
 PLATFORM_TEST_CHECK_ROLLUP_OK_SELECTOR = (
     platform_check_status_expr(
         PLATFORM_TEST_CHECK_ROLLUP_MATCHERS,
         branch_matcher=PLATFORM_TEST_PRIMARY_BRANCH_MATCHER,
         status_matcher=f'status=~"{PLATFORM_TEST_NON_FAILURE_STATUS}"',
+        window=None,
     )
 )
 PLATFORM_TEST_CHECK_ROLLUP_FAILED_SELECTOR = (
@@ -741,6 +757,7 @@ PLATFORM_TEST_CHECK_ROLLUP_FAILED_SELECTOR = (
         PLATFORM_TEST_CHECK_ROLLUP_MATCHERS,
         branch_matcher=PLATFORM_TEST_PRIMARY_BRANCH_MATCHER,
         status_matcher=f'status!~"{PLATFORM_TEST_NON_FAILURE_STATUS}"',
+        window=None,
     )
 )
 PLATFORM_TEST_CHECK_ROLLUP_SEEN_FLAGS = (
@@ -3832,6 +3849,7 @@ def build_jobs_dashboard():
             branch_matcher=f'branch!="",branch=~"{branch_var}"',
             check_matcher=check_matcher,
             status_matcher=status_matcher,
+            window=None,
         )
         return selector
 

scripts/tests/test_dashboards_render_atlas.py

@@ -229,10 +229,12 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     assert 'check)' in current_gate_expr
     assert "platform_quality:check_status:present_1h" not in current_gate_expr
     assert '.*_quality_gate_checks_total' in current_gate_expr
-    assert "last_over_time" in current_gate_expr
+    assert "last_over_time" not in current_gate_expr
     assert 'label_replace' in current_gate_expr
     assert 'result=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
     assert 'status=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
+    assert 'result!="",result!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
+    assert 'status!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
     assert 'status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
     assert "unless on(suite, check)" in current_gate_expr
     assert "tlast_over_time" not in current_gate_expr
@@ -342,8 +344,10 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
     assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
     assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"]
     assert '.*_quality_gate_checks_total' in failure_rate_panel["targets"][0]["expr"]
-    assert "last_over_time" in failure_rate_panel["targets"][0]["expr"]
+    assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
     assert 'label_replace' in failure_rate_panel["targets"][0]["expr"]
+    assert 'status!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in failure_rate_panel["targets"][0]["expr"]
+    assert 'result!="",result!~"ok|passed|success|not_applicable|skipped|na|n/a"' in failure_rate_panel["targets"][0]["expr"]
     assert "0 *" in failure_rate_panel["targets"][0]["expr"]
     assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]