diff --git a/ci/scripts/publish_test_metrics.py b/ci/scripts/publish_test_metrics.py index adafb018..473dc40f 100644 --- a/ci/scripts/publish_test_metrics.py +++ b/ci/scripts/publish_test_metrics.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import json import os from glob import glob @@ -27,6 +28,8 @@ _infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_p _load_optional_json = _quality_helpers._load_optional_json _normalize_result_status = _quality_helpers._normalize_result_status +TEST_CASE_LABEL_MAX_BYTES = 240 + def _escape_label(value: str) -> str: """Escape a Prometheus label value without changing its content.""" @@ -39,6 +42,17 @@ def _label_str(labels: dict[str, str]) -> str: return "{" + ",".join(parts) + "}" if parts else "" +def _bounded_test_name(value: str) -> str: + """Keep test labels readable, unique, and safely below scraper limits.""" + encoded = value.encode("utf-8") + if len(encoded) <= TEST_CASE_LABEL_MAX_BYTES: + return value + digest = hashlib.sha256(encoded).hexdigest()[:16] + suffix = f"...[sha256:{digest}]" + prefix = encoded[: TEST_CASE_LABEL_MAX_BYTES - len(suffix)] + return prefix.decode("utf-8", errors="ignore") + suffix + + def _read_text(url: str) -> str: """Fetch a plain-text response body from the given URL.""" with urllib.request.urlopen(url, timeout=10) as response: @@ -121,7 +135,7 @@ def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]: status = "failed" elif test_case.find("skipped") is not None: status = "skipped" - cases.append((full_name, status)) + cases.append((_bounded_test_name(full_name), status)) return cases @@ -246,7 +260,7 @@ def _build_payload( for test_name, test_status in test_cases: labels = { **test_case_base_labels, - "test": test_name, + "test": _bounded_test_name(test_name), "status": test_status, } lines.append( diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index 762d59bd..4a781030 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -1632,7 +1632,7 @@ def testing_case_variable(): "label": "Test Case", "type": "query", "query": ( - "query_result(topk(75, count by (test) (max_over_time(" + "query_result(bottomk(75, min by (test) (min_over_time(" f'platform_quality:test_case_health_rate:percent_1h{{suite=~"${{suite:regex}}",branch!="",' f'branch=~"${{branch:regex}}",test!="",test!="__no_test_cases__",' f'category!~"{PLATFORM_TEST_SUPPORT_CATEGORY_REGEX}"}}' diff --git a/scripts/tests/test_dashboards_render_jobs.py b/scripts/tests/test_dashboards_render_jobs.py index 8c91343b..46605f92 100644 --- a/scripts/tests/test_dashboards_render_jobs.py +++ b/scripts/tests/test_dashboards_render_jobs.py @@ -46,6 +46,8 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability(): test_variable = variables_by_name["test"] assert test_variable["query"].startswith("query_result(") + assert "bottomk(75" in test_variable["query"] + assert "min_over_time" in test_variable["query"] assert "platform_quality:test_case_health_rate:percent_1h" in test_variable["query"] assert "platform_quality_gate_test_case_result" not in test_variable["query"] assert test_variable["regex"] == '/test="([^"]+)"/' diff --git a/services/monitoring/dashboards/atlas-testing.json b/services/monitoring/dashboards/atlas-testing.json index 94eec73a..cc6f97f4 100644 --- a/services/monitoring/dashboards/atlas-testing.json +++ b/services/monitoring/dashboards/atlas-testing.json @@ -5044,7 +5044,7 @@ "name": "test", "label": "Test Case", "type": "query", - "query": "query_result(topk(75, count by (test) (max_over_time(platform_quality:test_case_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",category!~\"fixtures|golden|helpers\"}[24h:1h]))))", + "query": "query_result(bottomk(75, min by (test) (min_over_time(platform_quality:test_case_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",category!~\"fixtures|golden|helpers\"}[24h:1h]))))", "regex": "/test=\"([^\"]+)\"/", "current": { "text": "All", diff --git a/services/monitoring/grafana-dashboard-testing.yaml b/services/monitoring/grafana-dashboard-testing.yaml index c5d1369f..a63c7e95 100644 --- a/services/monitoring/grafana-dashboard-testing.yaml +++ b/services/monitoring/grafana-dashboard-testing.yaml @@ -5053,7 +5053,7 @@ data: "name": "test", "label": "Test Case", "type": "query", - "query": "query_result(topk(75, count by (test) (max_over_time(platform_quality:test_case_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",category!~\"fixtures|golden|helpers\"}[24h:1h]))))", + "query": "query_result(bottomk(75, min by (test) (min_over_time(platform_quality:test_case_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",category!~\"fixtures|golden|helpers\"}[24h:1h]))))", "regex": "/test=\"([^\"]+)\"/", "current": { "text": "All", diff --git a/testing/tests/test_hermes_deadline_http.py b/testing/tests/test_hermes_deadline_http.py index 45fd07e4..11411870 100644 --- a/testing/tests/test_hermes_deadline_http.py +++ b/testing/tests/test_hermes_deadline_http.py @@ -247,6 +247,16 @@ def test_exchange_returns_result_and_enforces_response_limit(): (b'{"ok":true,"status":"200","content_type":"a","body":""}', 0, "invalid evidence"), (b'{"ok":true,"status":200,"content_type":"a","body":"!!"}', 0, "invalid evidence"), ], + ids=[ + "empty-error", + "oversized-output", + "invalid-json", + "false-result", + "array-result", + "missing-fields", + "empty-body", + "invalid-body", + ], ) def test_exchange_rejects_invalid_helper_evidence(output, returncode, match): module = _load("deadline_http") diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py index 48421f01..c9026b5e 100644 --- a/testing/tests/test_hermes_gitea_branch_coverage.py +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -141,6 +141,7 @@ def test_created_at_requires_short_timezone_aware_iso(value): @pytest.mark.parametrize( "value", [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], + ids=["oversized-input", "object-input", "null-list-input"], ) def test_evaluate_rejects_invalid_top_level_inputs(value): module = _module("branch_evaluate_top") diff --git a/testing/tests/test_publish_test_metrics_paths.py b/testing/tests/test_publish_test_metrics_paths.py index 50d45cd8..f1585d77 100644 --- a/testing/tests/test_publish_test_metrics_paths.py +++ b/testing/tests/test_publish_test_metrics_paths.py @@ -65,6 +65,30 @@ def test_build_payload_includes_explicit_test_case_series(): ) +def test_test_case_labels_are_bounded_and_collision_safe(): + first = "module.test_case[" + ("x" * (4 * 1024 * 1024)) + "]" + second = first[:-2] + "y]" + + first_label = publish_test_metrics._bounded_test_name(first) + second_label = publish_test_metrics._bounded_test_name(second) + payload = publish_test_metrics._build_payload( + suite="titan_iac", + status="ok", + tests={"tests": 2, "failures": 0, "errors": 0, "skipped": 0}, + test_cases=[(first, "passed"), (second, "passed")], + ok_count=1, + failed_count=0, + branch="main", + build_number="1", + jenkins_job="titan-iac", + ) + + assert len(first_label.encode("utf-8")) <= publish_test_metrics.TEST_CASE_LABEL_MAX_BYTES + assert first_label != second_label + assert "[sha256:" in first_label + assert max(len(line.encode("utf-8")) for line in payload.splitlines()) < 1024 + + def test_main_uses_reported_coverage_and_loc_without_fallback(tmp_path: Path, monkeypatch, capsys): build_dir = tmp_path / "build" build_dir.mkdir()