bstein/ariadne
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(triage): report actionable incidents only

Browse Source
This commit is contained in:
codex 2026-08-03 03:58:05 -03:00
parent 9d90700d54
commit c8daa610cf
7 changed files with 174 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ariadne/services/cluster_state_pods.py
View File

@ -279,7 +279,9 @@ def _update_pod_issue(pod: dict[str, Any], acc: dict[str, Any]) -> None:
acc["phase_reasons"][phase_reason] = acc["phase_reasons"].get(phase_reason, 0) + 1
if phase in acc["counts"]:
acc["counts"][phase] += 1
if phase in _PHASE_SEVERITY or restarts > 0:
# Lifetime restart totals are historical context, not proof of a current
# incident. Current restart rates are collected separately from metrics.
if phase in _PHASE_SEVERITY or waiting_reasons:
acc["items"].append(
{
"namespace": namespace,

52
ariadne/services/testing_triage.py
View File

@ -30,6 +30,9 @@ logger = get_logger(__name__)
TRIAGE_EVENT_TYPE = "testing_triage_bundle"
_SUCCESS_STATUS = "ok|passed|success|not_applicable|skipped|na|n/a"
_MAX_EVIDENCE_ITEMS = 12
_RECENT_FAILED_JOB_HOURS = 24.0
_PENDING_GRACE_HOURS = 0.25
_LEGACY_NAMESPACES = frozenset({"veles"})
@dataclass(frozen=True)
@ -187,6 +190,9 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]:
if isinstance(snapshot.get("nodes_summary"), dict)
else {}
)
flux_items = _limit(flux.get("items"))
pod_items = _limit(pod_issues.get("items"))
job_items = _limit(jobs.get("failing"))
return {
"collected_at": snapshot.get("collected_at") or "",
"health_bullets": _limit(summary.get("health_bullets")),
@ -197,15 +203,52 @@ def _cluster_evidence(snapshot: dict[str, Any]) -> dict[str, Any]:
"not_ready": nodes.get("not_ready"),
"not_ready_names": nodes.get("not_ready_names") or [],
},
"flux_not_ready": _limit(flux.get("items")),
"pod_issues": _limit(pod_issues.get("items")),
"flux_not_ready": [item for item in flux_items if _flux_is_actionable(item)],
"flux_in_progress": [item for item in flux_items if _flux_is_in_progress(item)],
"pod_issues": [item for item in pod_items if _pod_is_actionable(item)],
"migration_residue": [item for item in pod_items if _is_legacy_item(item)],
"pending_oldest": _limit(pod_issues.get("pending_oldest")),
"jobs_failing": _limit(jobs.get("failing")),
"jobs_failing": [item for item in job_items if _job_is_recent(item)],
"jobs_active_oldest": _limit(jobs.get("active_oldest")),
"events_recent": _limit(events.get("warnings_recent")),
}
def _flux_is_actionable(item: Any) -> bool:
return isinstance(item, dict) and (
item.get("ready") is False or item.get("suspended") is True
)
def _flux_is_in_progress(item: Any) -> bool:
return isinstance(item, dict) and item.get("ready") is None
def _is_legacy_item(item: Any) -> bool:
return isinstance(item, dict) and item.get("namespace") in _LEGACY_NAMESPACES
def _pod_is_actionable(item: Any) -> bool:
if not isinstance(item, dict) or _is_legacy_item(item):
return False
phase = str(item.get("phase") or "")
if phase == "Pending":
return float(item.get("age_hours") or 0) >= _PENDING_GRACE_HOURS
if phase in {"Failed", "Unknown"}:
return _job_is_recent(item)
return bool(item.get("waiting_reasons"))
def _job_is_recent(item: Any) -> bool:
if not isinstance(item, dict):
return False
age = item.get("age_hours")
try:
return age is not None and float(age) <= _RECENT_FAILED_JOB_HOURS
except (TypeError, ValueError):
return False
def _quality_signals(errors: list[str]) -> dict[str, Any]:
queries = {
"failed_runs_24h": (
@ -320,6 +363,9 @@ def _render_markdown(bundle: dict[str, Any]) -> str:
*_markdown_items(cluster.get("health_bullets")),
*_markdown_named_items("Flux", cluster.get("flux_not_ready"), "name"),
*_markdown_named_items("Pods", cluster.get("pod_issues"), "pod"),
*_markdown_named_items(
"Migration residue", cluster.get("migration_residue"), "pod"
),
"",
"## Quality",
*_markdown_quality(quality),

13
ariadne/services/testing_triage_jenkins.py
View File

@ -38,23 +38,28 @@ def jenkins_signals(errors: list[str]) -> dict[str, Any]:
base_url = settings.jenkins_base_url.strip().rstrip("/")
if not base_url:
return {"failed_builds": []}
return {"failed_builds": [], "in_progress_builds": []}
try:
jobs = _fetch_jenkins_jobs(base_url)
except Exception as exc:
errors.append(f"jenkins: {exc}")
return {"failed_builds": []}
return {"failed_builds": [], "in_progress_builds": []}
scoped_jobs = [job for job in jobs if suite_in_scope(job.get("job"))]
failed = [
job
for job in scoped_jobs
if job.get("status") in {"failure", "running", "unknown"}
if job.get("status") in {"failure", "unknown"}
]
in_progress = [job for job in scoped_jobs if job.get("status") == "running"]
failed.sort(key=lambda item: -(item.get("last_run_ts") or 0))
in_progress.sort(key=lambda item: -(item.get("last_run_ts") or 0))
for job in failed[:3]:
_attach_jenkins_log_tail(job, errors)
_attach_jenkins_artifact_evidence(job, errors)
return {"failed_builds": failed[:_MAX_EVIDENCE_ITEMS]}
return {
"failed_builds": failed[:_MAX_EVIDENCE_ITEMS],
"in_progress_builds": in_progress[:_MAX_EVIDENCE_ITEMS],
}
def _fetch_jenkins_jobs(base_url: str) -> list[dict[str, Any]]:

10
ariadne/services/testing_triage_scope.py
View File

@ -10,6 +10,7 @@ IN_SCOPE_TEST_SUITES = frozenset(
"atlasbot",
"bstein_home",
"data_prepper",
"lesavka",
"metis",
"pegasus",
"soteria",
@ -75,8 +76,17 @@ def canonical_suite_name(value: Any) -> str:
def _quality_item_in_scope(item: dict[str, Any]) -> bool:
if _metric_value(item) <= 0:
return False
labels = item.get("labels") if isinstance(item, dict) else {}
if not isinstance(labels, dict):
return True
raw_suite = labels.get("suite") or labels.get("exported_job") or labels.get("job")
return suite_in_scope(raw_suite) if raw_suite else True
def _metric_value(item: dict[str, Any]) -> float:
try:
return float(item.get("value") or 0)
except (TypeError, ValueError):
return 0.0

89
tests/test_testing_triage.py
View File

@ -17,7 +17,15 @@ class DummyStorage:
"attention_ranked": [{"kind": "pod_pending"}],
},
"nodes_summary": {"total": 3, "ready": 3, "not_ready": 0},
"flux": {"items": [{"namespace": "flux-system", "name": "monitoring"}]},
"flux": {
"items": [
{
"namespace": "flux-system",
"name": "monitoring",
"ready": False,
}
]
},
"pod_issues": {
"items": [
{"namespace": "jenkins", "pod": "agent-1", "phase": "Pending"}
@ -192,12 +200,28 @@ def test_cluster_evidence_limits_and_defaults() -> None:
"not_ready": 1,
"not_ready_names": ["titan-06"],
},
"flux": {"items": [{"name": str(i)} for i in range(20)]},
"flux": {
"items": [
{"name": str(i), "ready": False if i < 10 else None}
for i in range(20)
]
},
"pod_issues": {
"items": [{"pod": str(i)} for i in range(20)],
"items": [
{
"namespace": "apps",
"pod": str(i),
"phase": "Pending",
"age_hours": 1,
}
for i in range(20)
],
"pending_oldest": [{"pod": "p"}],
},
"jobs": {"failing": [{"job": "j"}], "active_oldest": [{"job": "old"}]},
"jobs": {
"failing": [{"job": "j", "age_hours": 1}],
"active_oldest": [{"job": "old"}],
},
"events": {"warnings_recent": [{"message": "warn"}]},
}
@ -205,7 +229,62 @@ def test_cluster_evidence_limits_and_defaults() -> None:
assert len(evidence["health_bullets"]) == testing_triage._MAX_EVIDENCE_ITEMS # noqa: SLF001
assert evidence["nodes"]["not_ready_names"] == ["titan-06"]
assert evidence["jobs_failing"] == [{"job": "j"}]
assert evidence["jobs_failing"] == [{"job": "j", "age_hours": 1}]
def test_cluster_evidence_separates_active_incidents_from_context() -> None:
snapshot = {
"flux": {
"items": [
{"name": "ready-unknown", "ready": None},
{"name": "broken", "ready": False},
]
},
"pod_issues": {
"items": [
{
"namespace": "apps",
"pod": "old-restarts",
"phase": "Running",
"restarts": 100,
},
{
"namespace": "apps",
"pod": "starting",
"phase": "Pending",
"age_hours": 0.1,
},
{
"namespace": "apps",
"pod": "stuck",
"phase": "Pending",
"age_hours": 1,
},
{
"namespace": "veles",
"pod": "legacy",
"phase": "Pending",
"age_hours": 100,
},
]
},
"jobs": {
"failing": [
{"job": "recent", "age_hours": 1},
{"job": "historical", "age_hours": 100},
]
},
}
evidence = testing_triage._cluster_evidence(snapshot) # noqa: SLF001
assert evidence["flux_not_ready"] == [{"name": "broken", "ready": False}]
assert evidence["flux_in_progress"] == [
{"name": "ready-unknown", "ready": None}
]
assert [item["pod"] for item in evidence["pod_issues"]] == ["stuck"]
assert [item["pod"] for item in evidence["migration_residue"]] == ["legacy"]
assert evidence["jobs_failing"] == [{"job": "recent", "age_hours": 1}]
def test_vm_items_handles_success_failure_and_bad_values(monkeypatch) -> None:

14
tests/test_testing_triage_jenkins.py
View File

@ -317,7 +317,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None:
monkeypatch.setattr(
testing_triage_jenkins, "settings", SettingsStub(jenkins_base_url="")
)
assert testing_triage_jenkins.jenkins_signals([]) == {"failed_builds": []}
assert testing_triage_jenkins.jenkins_signals([]) == {
"failed_builds": [],
"in_progress_builds": [],
}
monkeypatch.setattr(
testing_triage_jenkins,
@ -331,7 +334,10 @@ def test_jenkins_signals_handles_disabled_and_failures(monkeypatch) -> None:
)
errors: list[str] = []
assert testing_triage_jenkins.jenkins_signals(errors) == {"failed_builds": []}
assert testing_triage_jenkins.jenkins_signals(errors) == {
"failed_builds": [],
"in_progress_builds": [],
}
assert errors == ["jenkins: boom"]
@ -360,8 +366,8 @@ def test_jenkins_signals_attaches_recent_failed_builds(monkeypatch) -> None:
signals = testing_triage_jenkins.jenkins_signals([])
assert [item["job"] for item in signals["failed_builds"]] == [
"soteria",
"metis",
"ariadne",
]
assert attached == ["soteria", "metis", "ariadne"]
assert [item["job"] for item in signals["in_progress_builds"]] == ["soteria"]
assert attached == ["metis", "ariadne"]

13
tests/test_testing_triage_scope.py
View File

@ -17,7 +17,7 @@ class SettingsStub:
def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None:
jobs = [
{"job": "lesavka", "status": "running", "last_run_ts": 50},
{"job": "lesavka", "status": "failure", "last_run_ts": 50},
{"job": "harbor-arm-build", "status": "failure", "last_run_ts": 40},
{"job": "data-prepper", "status": "running", "last_run_ts": 30},
{"job": "folder/ariadne", "status": "failure", "last_run_ts": 20},
@ -41,17 +41,21 @@ def test_jenkins_signals_filters_to_in_scope_suite_jobs(monkeypatch) -> None:
signals = testing_triage_jenkins.jenkins_signals([])
assert [item["job"] for item in signals["failed_builds"]] == [
"data-prepper",
"lesavka",
"folder/ariadne",
"bstein-dev-home",
]
assert attached == ["data-prepper", "folder/ariadne", "bstein-dev-home"]
assert [item["job"] for item in signals["in_progress_builds"]] == [
"data-prepper"
]
assert attached == ["lesavka", "folder/ariadne", "bstein-dev-home"]
def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None:
rows = [
{"labels": {"suite": "ariadne"}, "value": 1.0},
{"labels": {"suite": "lesavka"}, "value": 1.0},
{"labels": {"suite": "soteria"}, "value": 0.0},
{"labels": {"suite": "typhon"}, "value": 1.0},
{"labels": {"exported_job": "titan-iac"}, "value": 1.0},
{"labels": {"exported_job": "harbor-arm-build"}, "value": 1.0},
@ -64,7 +68,8 @@ def test_quality_signals_filters_to_in_scope_suites(monkeypatch) -> None:
assert quality["failed_runs_24h"]["items"] == [
{"labels": {"suite": "ariadne"}, "value": 1.0},
{"labels": {"suite": "lesavka"}, "value": 1.0},
{"labels": {"exported_job": "titan-iac"}, "value": 1.0},
{"labels": {}, "value": 1.0},
]
assert testing_triage._failed_suites(quality) == {"ariadne"} # noqa: SLF001
assert testing_triage._failed_suites(quality) == {"ariadne", "lesavka"} # noqa: SLF001