fix(triage): report actionable incidents only

ariadne/services/cluster_state_pods.py

@@ -279,7 +279,9 @@ def _update_pod_issue(pod: dict[str, Any], acc: dict[str, Any]) -> None:
         acc["phase_reasons"][phase_reason] = acc["phase_reasons"].get(phase_reason, 0) + 1
     if phase in acc["counts"]:
         acc["counts"][phase] += 1
-    if phase in _PHASE_SEVERITY or restarts > 0:
+    # Lifetime restart totals are historical context, not proof of a current
+    # incident. Current restart rates are collected separately from metrics.
+    if phase in _PHASE_SEVERITY or waiting_reasons:
         acc["items"].append(
             {
                 "namespace": namespace,

ariadne/services/testing_triage.py

@@ -30,6 +30,9 @@ logger = get_logger(__name__)
 TRIAGE_EVENT_TYPE = "testing_triage_bundle"
 _SUCCESS_STATUS = "ok|passed|success|not_applicable|skipped|na|n/a"
 _MAX_EVIDENCE_ITEMS = 12
+_RECENT_FAILED_JOB_HOURS = 24.0
+_PENDING_GRACE_HOURS = 0.25
+_LEGACY_NAMESPACES = frozenset({"veles"})
 
 
 @dataclass(frozen=True)
@@ -187,6 +190,9 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]:
         if isinstance(snapshot.get("nodes_summary"), dict)
         else {}
     )
+    flux_items = _limit(flux.get("items"))
+    pod_items = _limit(pod_issues.get("items"))
+    job_items = _limit(jobs.get("failing"))
     return {
         "collected_at": snapshot.get("collected_at") or "",
         "health_bullets": _limit(summary.get("health_bullets")),
@@ -197,15 +203,52 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]:
             "not_ready": nodes.get("not_ready"),
             "not_ready_names": nodes.get("not_ready_names") or [],
         },
-        "flux_not_ready": _limit(flux.get("items")),
+        "flux_not_ready": [item for item in flux_items if _flux_is_actionable(item)],
-        "pod_issues": _limit(pod_issues.get("items")),
+        "flux_in_progress": [item for item in flux_items if _flux_is_in_progress(item)],
+        "pod_issues": [item for item in pod_items if _pod_is_actionable(item)],
+        "migration_residue": [item for item in pod_items if _is_legacy_item(item)],
         "pending_oldest": _limit(pod_issues.get("pending_oldest")),
-        "jobs_failing": _limit(jobs.get("failing")),
+        "jobs_failing": [item for item in job_items if _job_is_recent(item)],
         "jobs_active_oldest": _limit(jobs.get("active_oldest")),
         "events_recent": _limit(events.get("warnings_recent")),
     }
 
 
+def _flux_is_actionable(item: Any) -> bool:
+    return isinstance(item, dict) and (
+        item.get("ready") is False or item.get("suspended") is True
+    )
+
+
+def _flux_is_in_progress(item: Any) -> bool:
+    return isinstance(item, dict) and item.get("ready") is None
+
+
+def _is_legacy_item(item: Any) -> bool:
+    return isinstance(item, dict) and item.get("namespace") in _LEGACY_NAMESPACES
+
+
+def _pod_is_actionable(item: Any) -> bool:
+    if not isinstance(item, dict) or _is_legacy_item(item):
+        return False
+    phase = str(item.get("phase") or "")
+    if phase == "Pending":
+        return float(item.get("age_hours") or 0) >= _PENDING_GRACE_HOURS
+    if phase in {"Failed", "Unknown"}:
+        return _job_is_recent(item)
+    return bool(item.get("waiting_reasons"))
+
+
+def _job_is_recent(item: Any) -> bool:
+    if not isinstance(item, dict):
+        return False
+    age = item.get("age_hours")
+    try:
+        return age is not None and float(age) <= _RECENT_FAILED_JOB_HOURS
+    except (TypeError, ValueError):
+        return False
+
+
 def _quality_signals(errors: list[str]) -> dict[str, Any]:
     queries = {
         "failed_runs_24h": (
@@ -320,6 +363,9 @@ def _render_markdown(bundle: dict[str, Any]) -> str:
         *_markdown_items(cluster.get("health_bullets")),
         *_markdown_named_items("Flux", cluster.get("flux_not_ready"), "name"),
         *_markdown_named_items("Pods", cluster.get("pod_issues"), "pod"),
+        *_markdown_named_items(
+            "Migration residue", cluster.get("migration_residue"), "pod"
+        ),
         "",
         "## Quality",
         *_markdown_quality(quality),

ariadne/services/testing_triage_jenkins.py

@@ -38,23 +38,28 @@ def jenkins_signals(errors: list[str]) -> dict[str, Any]:
 
     base_url = settings.jenkins_base_url.strip().rstrip("/")
     if not base_url:
-        return {"failed_builds": []}
+        return {"failed_builds": [], "in_progress_builds": []}
     try:
         jobs = _fetch_jenkins_jobs(base_url)
     except Exception as exc:
         errors.append(f"jenkins: {exc}")
-        return {"failed_builds": []}
+        return {"failed_builds": [], "in_progress_builds": []}
     scoped_jobs = [job for job in jobs if suite_in_scope(job.get("job"))]
     failed = [
         job
         for job in scoped_jobs
-        if job.get("status") in {"failure", "running", "unknown"}
+        if job.get("status") in {"failure", "unknown"}
     ]
+    in_progress = [job for job in scoped_jobs if job.get("status") == "running"]
     failed.sort(key=lambda item: -(item.get("last_run_ts") or 0))
+    in_progress.sort(key=lambda item: -(item.get("last_run_ts") or 0))
     for job in failed[:3]:
         _attach_jenkins_log_tail(job, errors)
         _attach_jenkins_artifact_evidence(job, errors)
-    return {"failed_builds": failed[:_MAX_EVIDENCE_ITEMS]}
+    return {
+        "failed_builds": failed[:_MAX_EVIDENCE_ITEMS],
+        "in_progress_builds": in_progress[:_MAX_EVIDENCE_ITEMS],
+    }
 
 
 def _fetch_jenkins_jobs(base_url: str) -> list[dict[str, Any]]:

ariadne/services/testing_triage_scope.py

@@ -10,6 +10,7 @@ IN_SCOPE_TEST_SUITES = frozenset(
         "atlasbot",
         "bstein_home",
         "data_prepper",
+        "lesavka",
         "metis",
         "pegasus",
         "soteria",
@@ -75,8 +76,17 @@ def canonical_suite_name(value: Any) -> str:
 
 
 def _quality_item_in_scope(item: dict[str, Any]) -> bool:
+    if _metric_value(item) <= 0:
+        return False
     labels = item.get("labels") if isinstance(item, dict) else {}
     if not isinstance(labels, dict):
         return True
     raw_suite = labels.get("suite") or labels.get("exported_job") or labels.get("job")
     return suite_in_scope(raw_suite) if raw_suite else True
+
+
+def _metric_value(item: dict[str, Any]) -> float:
+    try:
+        return float(item.get("value") or 0)
+    except (TypeError, ValueError):
+        return 0.0

tests/test_testing_triage.py

@@ -17,7 +17,15 @@ class DummyStorage:
                 "attention_ranked": [{"kind": "pod_pending"}],
             },
             "nodes_summary": {"total": 3, "ready": 3, "not_ready": 0},
-            "flux": {"items": [{"namespace": "flux-system", "name": "monitoring"}]},
+            "flux": {
+                "items": [
+                    {
+                        "namespace": "flux-system",
+                        "name": "monitoring",
+                        "ready": False,
+                    }
+                ]
+            },
             "pod_issues": {
                 "items": [
                     {"namespace": "jenkins", "pod": "agent-1", "phase": "Pending"}
@@ -192,12 +200,28 @@ def test_cluster_evidence_limits_and_defaults() -> None:
             "not_ready": 1,
             "not_ready_names": ["titan-06"],
         },
-        "flux": {"items": [{"name": str(i)} for i in range(20)]},
+        "flux": {
+            "items": [
+                {"name": str(i), "ready": False if i < 10 else None}
+                for i in range(20)
+            ]
+        },
         "pod_issues": {
-            "items": [{"pod": str(i)} for i in range(20)],
+            "items": [
+                {
+                    "namespace": "apps",
+                    "pod": str(i),
+                    "phase": "Pending",
+                    "age_hours": 1,
+                }
+                for i in range(20)
+            ],
             "pending_oldest": [{"pod": "p"}],
         },
-        "jobs": {"failing": [{"job": "j"}], "active_oldest": [{"job": "old"}]},
+        "jobs": {
+            "failing": [{"job": "j", "age_hours": 1}],
+            "active_oldest": [{"job": "old"}],
+        },
         "events": {"warnings_recent": [{"message": "warn"}]},
     }
 
@@ -205,7 +229,62 @@ def test_cluster_evidence_limits_and_defaults() -> None:
 
     assert len(evidence["health_bullets"]) == testing_triage._MAX_EVIDENCE_ITEMS  # noqa: SLF001
     assert evidence["nodes"]["not_ready_names"] == ["titan-06"]
-    assert evidence["jobs_failing"] == [{"job": "j"}]
+    assert evidence["jobs_failing"] == [{"job": "j", "age_hours": 1}]
+
+
+def test_cluster_evidence_separates_active_incidents_from_context() -> None:
+    snapshot = {
+        "flux": {
+            "items": [
+                {"name": "ready-unknown", "ready": None},
+                {"name": "broken", "ready": False},
+            ]
+        },
+        "pod_issues": {
+            "items": [
+                {
+                    "namespace": "apps",
+                    "pod": "old-restarts",
+                    "phase": "Running",
+                    "restarts": 100,
+                },
+                {
+                    "namespace": "apps",
+                    "pod": "starting",
+                    "phase": "Pending",
+                    "age_hours": 0.1,
+                },
+                {
+                    "namespace": "apps",
+                    "pod": "stuck",
+                    "phase": "Pending",
+                    "age_hours": 1,
+                },
+                {
+                    "namespace": "veles",
+                    "pod": "legacy",
+                    "phase": "Pending",
+                    "age_hours": 100,
+                },
+            ]
+        },
+        "jobs": {
+            "failing": [
+                {"job": "recent", "age_hours": 1},
+                {"job": "historical", "age_hours": 100},
+            ]
+        },
+    }
+
+    evidence = testing_triage._cluster_evidence(snapshot)  # noqa: SLF001
+
+    assert evidence["flux_not_ready"] == [{"name": "broken", "ready": False}]
+    assert evidence["flux_in_progress"] == [
+        {"name": "ready-unknown", "ready": None}
+    ]
+    assert [item["pod"] for item in evidence["pod_issues"]] == ["stuck"]
+    assert [item["pod"] for item in evidence["migration_residue"]] == ["legacy"]
+    assert evidence["jobs_failing"] == [{"job": "recent", "age_hours": 1}]
 
 
 def test_vm_items_handles_success_failure_and_bad_values(monkeypatch) -> None:

tests/test_testing_triage_jenkins.py

@@ -317,7 +317,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None:
     monkeypatch.setattr(
         testing_triage_jenkins, "settings", SettingsStub(jenkins_base_url="")
     )
-    assert testing_triage_jenkins.jenkins_signals([]) == {"failed_builds": []}
+    assert testing_triage_jenkins.jenkins_signals([]) == {
+        "failed_builds": [],
+        "in_progress_builds": [],
+    }
 
     monkeypatch.setattr(
         testing_triage_jenkins,
@@ -331,7 +334,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None:
     )
     errors: list[str] = []
 
-    assert testing_triage_jenkins.jenkins_signals(errors) == {"failed_builds": []}
+    assert testing_triage_jenkins.jenkins_signals(errors) == {
+        "failed_builds": [],
+        "in_progress_builds": [],
+    }
     assert errors == ["jenkins: boom"]
 
 
@@ -360,8 +366,8 @@ def test_jenkins_signals_attaches_recent_failed_builds(monkeypatch) -> None:
     signals = testing_triage_jenkins.jenkins_signals([])
 
     assert [item["job"] for item in signals["failed_builds"]] == [
-        "soteria",
         "metis",
         "ariadne",
     ]
-    assert attached == ["soteria", "metis", "ariadne"]
+    assert [item["job"] for item in signals["in_progress_builds"]] == ["soteria"]
+    assert attached == ["metis", "ariadne"]

tests/test_testing_triage_scope.py

@@ -17,7 +17,7 @@ class SettingsStub:
 
 def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None:
     jobs = [
-        {"job": "lesavka", "status": "running", "last_run_ts": 50},
+        {"job": "lesavka", "status": "failure", "last_run_ts": 50},
         {"job": "harbor-arm-build", "status": "failure", "last_run_ts": 40},
         {"job": "data-prepper", "status": "running", "last_run_ts": 30},
         {"job": "folder/ariadne", "status": "failure", "last_run_ts": 20},
@@ -41,17 +41,21 @@ def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None:
     signals = testing_triage_jenkins.jenkins_signals([])
 
     assert [item["job"] for item in signals["failed_builds"]] == [
-        "data-prepper",
+        "lesavka",
         "folder/ariadne",
         "bstein-dev-home",
     ]
-    assert attached == ["data-prepper", "folder/ariadne", "bstein-dev-home"]
+    assert [item["job"] for item in signals["in_progress_builds"]] == [
+        "data-prepper"
+    ]
+    assert attached == ["lesavka", "folder/ariadne", "bstein-dev-home"]
 
 
 def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None:
     rows = [
         {"labels": {"suite": "ariadne"}, "value": 1.0},
         {"labels": {"suite": "lesavka"}, "value": 1.0},
+        {"labels": {"suite": "soteria"}, "value": 0.0},
         {"labels": {"suite": "typhon"}, "value": 1.0},
         {"labels": {"exported_job": "titan-iac"}, "value": 1.0},
         {"labels": {"exported_job": "harbor-arm-build"}, "value": 1.0},
@@ -64,7 +68,8 @@ def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None:
 
     assert quality["failed_runs_24h"]["items"] == [
         {"labels": {"suite": "ariadne"}, "value": 1.0},
+        {"labels": {"suite": "lesavka"}, "value": 1.0},
         {"labels": {"exported_job": "titan-iac"}, "value": 1.0},
         {"labels": {}, "value": 1.0},
     ]
-    assert testing_triage._failed_suites(quality) == {"ariadne"}  # noqa: SLF001
+    assert testing_triage._failed_suites(quality) == {"ariadne", "lesavka"}  # noqa: SLF001