From c8daa610cf6556befca1aa921f26637f849b3f03 Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 3 Aug 2026 03:58:05 -0300 Subject: [PATCH] fix(triage): report actionable incidents only --- ariadne/services/cluster_state_pods.py | 4 +- ariadne/services/testing_triage.py | 52 ++++++++++++- ariadne/services/testing_triage_jenkins.py | 13 +++- ariadne/services/testing_triage_scope.py | 10 +++ tests/test_testing_triage.py | 89 ++++++++++++++++++++-- tests/test_testing_triage_jenkins.py | 14 +++- tests/test_testing_triage_scope.py | 13 +++- 7 files changed, 174 insertions(+), 21 deletions(-) diff --git a/ariadne/services/cluster_state_pods.py b/ariadne/services/cluster_state_pods.py index 9db91d9..66a0b2b 100644 --- a/ariadne/services/cluster_state_pods.py +++ b/ariadne/services/cluster_state_pods.py @@ -279,7 +279,9 @@ def _update_pod_issue(pod: dict[str, Any], acc: dict[str, Any]) -> None: acc["phase_reasons"][phase_reason] = acc["phase_reasons"].get(phase_reason, 0) + 1 if phase in acc["counts"]: acc["counts"][phase] += 1 - if phase in _PHASE_SEVERITY or restarts > 0: + # Lifetime restart totals are historical context, not proof of a current + # incident. Current restart rates are collected separately from metrics. + if phase in _PHASE_SEVERITY or waiting_reasons: acc["items"].append( { "namespace": namespace, diff --git a/ariadne/services/testing_triage.py b/ariadne/services/testing_triage.py index 02b7e7e..91d5734 100644 --- a/ariadne/services/testing_triage.py +++ b/ariadne/services/testing_triage.py @@ -30,6 +30,9 @@ logger = get_logger(__name__) TRIAGE_EVENT_TYPE = "testing_triage_bundle" _SUCCESS_STATUS = "ok|passed|success|not_applicable|skipped|na|n/a" _MAX_EVIDENCE_ITEMS = 12 +_RECENT_FAILED_JOB_HOURS = 24.0 +_PENDING_GRACE_HOURS = 0.25 +_LEGACY_NAMESPACES = frozenset({"veles"}) @dataclass(frozen=True) @@ -187,6 +190,9 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]: if isinstance(snapshot.get("nodes_summary"), dict) else {} ) + flux_items = _limit(flux.get("items")) + pod_items = _limit(pod_issues.get("items")) + job_items = _limit(jobs.get("failing")) return { "collected_at": snapshot.get("collected_at") or "", "health_bullets": _limit(summary.get("health_bullets")), @@ -197,15 +203,52 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]: "not_ready": nodes.get("not_ready"), "not_ready_names": nodes.get("not_ready_names") or [], }, - "flux_not_ready": _limit(flux.get("items")), - "pod_issues": _limit(pod_issues.get("items")), + "flux_not_ready": [item for item in flux_items if _flux_is_actionable(item)], + "flux_in_progress": [item for item in flux_items if _flux_is_in_progress(item)], + "pod_issues": [item for item in pod_items if _pod_is_actionable(item)], + "migration_residue": [item for item in pod_items if _is_legacy_item(item)], "pending_oldest": _limit(pod_issues.get("pending_oldest")), - "jobs_failing": _limit(jobs.get("failing")), + "jobs_failing": [item for item in job_items if _job_is_recent(item)], "jobs_active_oldest": _limit(jobs.get("active_oldest")), "events_recent": _limit(events.get("warnings_recent")), } +def _flux_is_actionable(item: Any) -> bool: + return isinstance(item, dict) and ( + item.get("ready") is False or item.get("suspended") is True + ) + + +def _flux_is_in_progress(item: Any) -> bool: + return isinstance(item, dict) and item.get("ready") is None + + +def _is_legacy_item(item: Any) -> bool: + return isinstance(item, dict) and item.get("namespace") in _LEGACY_NAMESPACES + + +def _pod_is_actionable(item: Any) -> bool: + if not isinstance(item, dict) or _is_legacy_item(item): + return False + phase = str(item.get("phase") or "") + if phase == "Pending": + return float(item.get("age_hours") or 0) >= _PENDING_GRACE_HOURS + if phase in {"Failed", "Unknown"}: + return _job_is_recent(item) + return bool(item.get("waiting_reasons")) + + +def _job_is_recent(item: Any) -> bool: + if not isinstance(item, dict): + return False + age = item.get("age_hours") + try: + return age is not None and float(age) <= _RECENT_FAILED_JOB_HOURS + except (TypeError, ValueError): + return False + + def _quality_signals(errors: list[str]) -> dict[str, Any]: queries = { "failed_runs_24h": ( @@ -320,6 +363,9 @@ def _render_markdown(bundle: dict[str, Any]) -> str: *_markdown_items(cluster.get("health_bullets")), *_markdown_named_items("Flux", cluster.get("flux_not_ready"), "name"), *_markdown_named_items("Pods", cluster.get("pod_issues"), "pod"), + *_markdown_named_items( + "Migration residue", cluster.get("migration_residue"), "pod" + ), "", "## Quality", *_markdown_quality(quality), diff --git a/ariadne/services/testing_triage_jenkins.py b/ariadne/services/testing_triage_jenkins.py index fabec10..db93a0e 100644 --- a/ariadne/services/testing_triage_jenkins.py +++ b/ariadne/services/testing_triage_jenkins.py @@ -38,23 +38,28 @@ def jenkins_signals(errors: list[str]) -> dict[str, Any]: base_url = settings.jenkins_base_url.strip().rstrip("/") if not base_url: - return {"failed_builds": []} + return {"failed_builds": [], "in_progress_builds": []} try: jobs = _fetch_jenkins_jobs(base_url) except Exception as exc: errors.append(f"jenkins: {exc}") - return {"failed_builds": []} + return {"failed_builds": [], "in_progress_builds": []} scoped_jobs = [job for job in jobs if suite_in_scope(job.get("job"))] failed = [ job for job in scoped_jobs - if job.get("status") in {"failure", "running", "unknown"} + if job.get("status") in {"failure", "unknown"} ] + in_progress = [job for job in scoped_jobs if job.get("status") == "running"] failed.sort(key=lambda item: -(item.get("last_run_ts") or 0)) + in_progress.sort(key=lambda item: -(item.get("last_run_ts") or 0)) for job in failed[:3]: _attach_jenkins_log_tail(job, errors) _attach_jenkins_artifact_evidence(job, errors) - return {"failed_builds": failed[:_MAX_EVIDENCE_ITEMS]} + return { + "failed_builds": failed[:_MAX_EVIDENCE_ITEMS], + "in_progress_builds": in_progress[:_MAX_EVIDENCE_ITEMS], + } def _fetch_jenkins_jobs(base_url: str) -> list[dict[str, Any]]: diff --git a/ariadne/services/testing_triage_scope.py b/ariadne/services/testing_triage_scope.py index 426d6bf..91ddd20 100644 --- a/ariadne/services/testing_triage_scope.py +++ b/ariadne/services/testing_triage_scope.py @@ -10,6 +10,7 @@ IN_SCOPE_TEST_SUITES = frozenset( "atlasbot", "bstein_home", "data_prepper", + "lesavka", "metis", "pegasus", "soteria", @@ -75,8 +76,17 @@ def canonical_suite_name(value: Any) -> str: def _quality_item_in_scope(item: dict[str, Any]) -> bool: + if _metric_value(item) <= 0: + return False labels = item.get("labels") if isinstance(item, dict) else {} if not isinstance(labels, dict): return True raw_suite = labels.get("suite") or labels.get("exported_job") or labels.get("job") return suite_in_scope(raw_suite) if raw_suite else True + + +def _metric_value(item: dict[str, Any]) -> float: + try: + return float(item.get("value") or 0) + except (TypeError, ValueError): + return 0.0 diff --git a/tests/test_testing_triage.py b/tests/test_testing_triage.py index 7276bd1..0eec69e 100644 --- a/tests/test_testing_triage.py +++ b/tests/test_testing_triage.py @@ -17,7 +17,15 @@ class DummyStorage: "attention_ranked": [{"kind": "pod_pending"}], }, "nodes_summary": {"total": 3, "ready": 3, "not_ready": 0}, - "flux": {"items": [{"namespace": "flux-system", "name": "monitoring"}]}, + "flux": { + "items": [ + { + "namespace": "flux-system", + "name": "monitoring", + "ready": False, + } + ] + }, "pod_issues": { "items": [ {"namespace": "jenkins", "pod": "agent-1", "phase": "Pending"} @@ -192,12 +200,28 @@ def test_cluster_evidence_limits_and_defaults() -> None: "not_ready": 1, "not_ready_names": ["titan-06"], }, - "flux": {"items": [{"name": str(i)} for i in range(20)]}, + "flux": { + "items": [ + {"name": str(i), "ready": False if i < 10 else None} + for i in range(20) + ] + }, "pod_issues": { - "items": [{"pod": str(i)} for i in range(20)], + "items": [ + { + "namespace": "apps", + "pod": str(i), + "phase": "Pending", + "age_hours": 1, + } + for i in range(20) + ], "pending_oldest": [{"pod": "p"}], }, - "jobs": {"failing": [{"job": "j"}], "active_oldest": [{"job": "old"}]}, + "jobs": { + "failing": [{"job": "j", "age_hours": 1}], + "active_oldest": [{"job": "old"}], + }, "events": {"warnings_recent": [{"message": "warn"}]}, } @@ -205,7 +229,62 @@ def test_cluster_evidence_limits_and_defaults() -> None: assert len(evidence["health_bullets"]) == testing_triage._MAX_EVIDENCE_ITEMS # noqa: SLF001 assert evidence["nodes"]["not_ready_names"] == ["titan-06"] - assert evidence["jobs_failing"] == [{"job": "j"}] + assert evidence["jobs_failing"] == [{"job": "j", "age_hours": 1}] + + +def test_cluster_evidence_separates_active_incidents_from_context() -> None: + snapshot = { + "flux": { + "items": [ + {"name": "ready-unknown", "ready": None}, + {"name": "broken", "ready": False}, + ] + }, + "pod_issues": { + "items": [ + { + "namespace": "apps", + "pod": "old-restarts", + "phase": "Running", + "restarts": 100, + }, + { + "namespace": "apps", + "pod": "starting", + "phase": "Pending", + "age_hours": 0.1, + }, + { + "namespace": "apps", + "pod": "stuck", + "phase": "Pending", + "age_hours": 1, + }, + { + "namespace": "veles", + "pod": "legacy", + "phase": "Pending", + "age_hours": 100, + }, + ] + }, + "jobs": { + "failing": [ + {"job": "recent", "age_hours": 1}, + {"job": "historical", "age_hours": 100}, + ] + }, + } + + evidence = testing_triage._cluster_evidence(snapshot) # noqa: SLF001 + + assert evidence["flux_not_ready"] == [{"name": "broken", "ready": False}] + assert evidence["flux_in_progress"] == [ + {"name": "ready-unknown", "ready": None} + ] + assert [item["pod"] for item in evidence["pod_issues"]] == ["stuck"] + assert [item["pod"] for item in evidence["migration_residue"]] == ["legacy"] + assert evidence["jobs_failing"] == [{"job": "recent", "age_hours": 1}] def test_vm_items_handles_success_failure_and_bad_values(monkeypatch) -> None: diff --git a/tests/test_testing_triage_jenkins.py b/tests/test_testing_triage_jenkins.py index e75bd98..ebd7052 100644 --- a/tests/test_testing_triage_jenkins.py +++ b/tests/test_testing_triage_jenkins.py @@ -317,7 +317,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None: monkeypatch.setattr( testing_triage_jenkins, "settings", SettingsStub(jenkins_base_url="") ) - assert testing_triage_jenkins.jenkins_signals([]) == {"failed_builds": []} + assert testing_triage_jenkins.jenkins_signals([]) == { + "failed_builds": [], + "in_progress_builds": [], + } monkeypatch.setattr( testing_triage_jenkins, @@ -331,7 +334,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None: ) errors: list[str] = [] - assert testing_triage_jenkins.jenkins_signals(errors) == {"failed_builds": []} + assert testing_triage_jenkins.jenkins_signals(errors) == { + "failed_builds": [], + "in_progress_builds": [], + } assert errors == ["jenkins: boom"] @@ -360,8 +366,8 @@ def test_jenkins_signals_attaches_recent_failed_builds(monkeypatch) -> None: signals = testing_triage_jenkins.jenkins_signals([]) assert [item["job"] for item in signals["failed_builds"]] == [ - "soteria", "metis", "ariadne", ] - assert attached == ["soteria", "metis", "ariadne"] + assert [item["job"] for item in signals["in_progress_builds"]] == ["soteria"] + assert attached == ["metis", "ariadne"] diff --git a/tests/test_testing_triage_scope.py b/tests/test_testing_triage_scope.py index 4e7fc95..61b47df 100644 --- a/tests/test_testing_triage_scope.py +++ b/tests/test_testing_triage_scope.py @@ -17,7 +17,7 @@ class SettingsStub: def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None: jobs = [ - {"job": "lesavka", "status": "running", "last_run_ts": 50}, + {"job": "lesavka", "status": "failure", "last_run_ts": 50}, {"job": "harbor-arm-build", "status": "failure", "last_run_ts": 40}, {"job": "data-prepper", "status": "running", "last_run_ts": 30}, {"job": "folder/ariadne", "status": "failure", "last_run_ts": 20}, @@ -41,17 +41,21 @@ def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None: signals = testing_triage_jenkins.jenkins_signals([]) assert [item["job"] for item in signals["failed_builds"]] == [ - "data-prepper", + "lesavka", "folder/ariadne", "bstein-dev-home", ] - assert attached == ["data-prepper", "folder/ariadne", "bstein-dev-home"] + assert [item["job"] for item in signals["in_progress_builds"]] == [ + "data-prepper" + ] + assert attached == ["lesavka", "folder/ariadne", "bstein-dev-home"] def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None: rows = [ {"labels": {"suite": "ariadne"}, "value": 1.0}, {"labels": {"suite": "lesavka"}, "value": 1.0}, + {"labels": {"suite": "soteria"}, "value": 0.0}, {"labels": {"suite": "typhon"}, "value": 1.0}, {"labels": {"exported_job": "titan-iac"}, "value": 1.0}, {"labels": {"exported_job": "harbor-arm-build"}, "value": 1.0}, @@ -64,7 +68,8 @@ def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None: assert quality["failed_runs_24h"]["items"] == [ {"labels": {"suite": "ariadne"}, "value": 1.0}, + {"labels": {"suite": "lesavka"}, "value": 1.0}, {"labels": {"exported_job": "titan-iac"}, "value": 1.0}, {"labels": {}, "value": 1.0}, ] - assert testing_triage._failed_suites(quality) == {"ariadne"} # noqa: SLF001 + assert testing_triage._failed_suites(quality) == {"ariadne", "lesavka"} # noqa: SLF001