425 lines
14 KiB
Python
425 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from ariadne.services import testing_triage
|
|
|
|
|
|
class DummyStorage:
|
|
def __init__(self) -> None:
|
|
self.events: list[tuple[str, dict]] = []
|
|
|
|
def latest_cluster_state(self): # type: ignore[no-untyped-def]
|
|
return {
|
|
"collected_at": "2026-05-20T00:00:00+00:00",
|
|
"summary": {
|
|
"health_bullets": ["Pods pending: 1"],
|
|
"attention_ranked": [{"kind": "pod_pending"}],
|
|
},
|
|
"nodes_summary": {"total": 3, "ready": 3, "not_ready": 0},
|
|
"flux": {
|
|
"items": [
|
|
{
|
|
"namespace": "flux-system",
|
|
"name": "monitoring",
|
|
"ready": False,
|
|
}
|
|
]
|
|
},
|
|
"pod_issues": {
|
|
"items": [
|
|
{"namespace": "jenkins", "pod": "agent-1", "phase": "Pending"}
|
|
],
|
|
"pending_oldest": [{"namespace": "jenkins", "pod": "agent-1"}],
|
|
},
|
|
"jobs": {"failing": [], "active_oldest": []},
|
|
"events": {"warnings_recent": []},
|
|
}
|
|
|
|
def record_event(self, event_type: str, detail: dict) -> None:
|
|
self.events.append((event_type, detail))
|
|
|
|
def list_events(self, limit: int = 1, event_type: str | None = None): # type: ignore[no-untyped-def]
|
|
matching = [
|
|
{"detail": detail}
|
|
for stored_type, detail in self.events
|
|
if event_type is None or stored_type == event_type
|
|
]
|
|
return matching[-limit:][::-1]
|
|
|
|
|
|
class SettingsStub:
|
|
def __init__(self, **overrides) -> None: # type: ignore[no-untyped-def]
|
|
self.vm_url = ""
|
|
self.cluster_state_vm_timeout_sec = 1.0
|
|
self.jenkins_base_url = ""
|
|
self.jenkins_api_user = ""
|
|
self.jenkins_api_token = ""
|
|
self.jenkins_api_timeout_sec = 1.0
|
|
self.testing_triage_model_url = ""
|
|
self.testing_triage_model = "qwen2.5:7b-instruct-q4_0"
|
|
self.testing_triage_model_timeout_sec = 1.0
|
|
for key, value in overrides.items():
|
|
setattr(self, key, value)
|
|
|
|
|
|
def test_collect_testing_triage_builds_bundle(monkeypatch) -> None:
|
|
storage = DummyStorage()
|
|
monkeypatch.setattr(
|
|
testing_triage,
|
|
"_vm_items",
|
|
lambda query, errors: [{"labels": {"suite": "ariadne"}, "value": 1.0}]
|
|
if "platform_quality_gate_runs_total" in query
|
|
else [],
|
|
)
|
|
monkeypatch.setattr(
|
|
testing_triage, "_jenkins_signals", lambda errors: {"failed_builds": []}
|
|
)
|
|
|
|
bundle = testing_triage.collect_testing_triage(storage)
|
|
|
|
assert bundle["kind"] == "testing_triage_bundle"
|
|
assert bundle["summary"]["status"] == "needs_attention"
|
|
assert bundle["summary"]["failed_suites"] == ["ariadne"]
|
|
assert "Testing Triage Evidence" in bundle["markdown"]
|
|
assert bundle["openclaw"]["ariadne_latest_url"].endswith(
|
|
"/api/internal/testing/triage/latest"
|
|
)
|
|
|
|
|
|
def test_run_testing_triage_stores_latest(monkeypatch) -> None:
|
|
storage = DummyStorage()
|
|
monkeypatch.setattr(
|
|
testing_triage,
|
|
"collect_testing_triage",
|
|
lambda _storage: {
|
|
"summary": {"status": "ok", "problem_count": 0, "failed_suites": []},
|
|
},
|
|
)
|
|
|
|
summary = testing_triage.run_testing_triage(storage)
|
|
latest = testing_triage.latest_testing_triage_bundle(storage)
|
|
|
|
assert summary.status == "ok"
|
|
assert storage.events[0][0] == testing_triage.TRIAGE_EVENT_TYPE
|
|
assert latest["summary"]["status"] == "ok"
|
|
|
|
|
|
def test_run_testing_triage_diagnosis_stores_bundle_and_result(monkeypatch) -> None:
|
|
storage = DummyStorage()
|
|
bundle = {"summary": {"status": "needs_attention"}}
|
|
diagnosis = {"status": "ok", "model": "test-model"}
|
|
monkeypatch.setattr(
|
|
testing_triage, "collect_testing_triage", lambda _storage: bundle
|
|
)
|
|
monkeypatch.setattr(
|
|
testing_triage, "diagnose_testing_triage", lambda _bundle: diagnosis
|
|
)
|
|
|
|
result = testing_triage.run_testing_triage_diagnosis(storage)
|
|
|
|
assert result == diagnosis
|
|
assert storage.events == [
|
|
(testing_triage.TRIAGE_EVENT_TYPE, bundle),
|
|
(testing_triage.TRIAGE_DIAGNOSIS_EVENT_TYPE, diagnosis),
|
|
]
|
|
|
|
|
|
def test_latest_testing_triage_bundle_handles_json_strings() -> None:
|
|
class JsonStorage:
|
|
def list_events(self, limit: int = 1, event_type: str | None = None): # type: ignore[no-untyped-def]
|
|
assert limit == 1
|
|
assert event_type == testing_triage.TRIAGE_EVENT_TYPE
|
|
return [{"detail": json.dumps({"summary": {"status": "ok"}})}]
|
|
|
|
latest = testing_triage.latest_testing_triage_bundle(JsonStorage()) # type: ignore[arg-type]
|
|
|
|
assert latest == {"summary": {"status": "ok"}}
|
|
|
|
|
|
def test_latest_testing_triage_bundle_ignores_bad_payloads() -> None:
|
|
class BadStorage:
|
|
def __init__(self, detail) -> None: # type: ignore[no-untyped-def]
|
|
self.detail = detail
|
|
|
|
def list_events(self, limit: int = 1, event_type: str | None = None): # type: ignore[no-untyped-def]
|
|
return [{"detail": self.detail}]
|
|
|
|
assert testing_triage.latest_testing_triage_bundle(BadStorage("{")) is None # type: ignore[arg-type]
|
|
assert testing_triage.latest_testing_triage_bundle(BadStorage(["nope"])) is None # type: ignore[arg-type]
|
|
assert testing_triage.latest_testing_triage_bundle(BadStorage(None)) is None # type: ignore[arg-type]
|
|
|
|
class EmptyStorage:
|
|
def list_events(self, limit: int = 1, event_type: str | None = None): # type: ignore[no-untyped-def]
|
|
return []
|
|
|
|
assert testing_triage.latest_testing_triage_bundle(EmptyStorage()) is None # type: ignore[arg-type]
|
|
|
|
|
|
def test_latest_cluster_snapshot_falls_back_to_live_collect(monkeypatch) -> None:
|
|
class BrokenStorage:
|
|
def latest_cluster_state(self): # type: ignore[no-untyped-def]
|
|
raise RuntimeError("storage down")
|
|
|
|
monkeypatch.setattr(
|
|
testing_triage,
|
|
"collect_cluster_state",
|
|
lambda: ({"collected_at": "live"}, {"status": "ok"}),
|
|
)
|
|
errors: list[str] = []
|
|
|
|
snapshot = testing_triage._latest_cluster_snapshot(BrokenStorage(), errors) # noqa: SLF001
|
|
|
|
assert snapshot == {"collected_at": "live"}
|
|
assert errors == ["cluster_state_latest: storage down"]
|
|
|
|
|
|
def test_latest_cluster_snapshot_records_collect_error(monkeypatch) -> None:
|
|
monkeypatch.setattr(
|
|
testing_triage,
|
|
"collect_cluster_state",
|
|
lambda: (_ for _ in ()).throw(RuntimeError("api down")),
|
|
)
|
|
errors: list[str] = []
|
|
|
|
snapshot = testing_triage._latest_cluster_snapshot(None, errors) # noqa: SLF001
|
|
|
|
assert snapshot == {}
|
|
assert errors == ["cluster_state_collect: api down"]
|
|
|
|
|
|
def test_cluster_evidence_limits_and_defaults() -> None:
|
|
snapshot = {
|
|
"summary": {
|
|
"health_bullets": list(range(20)),
|
|
"attention_ranked": [{"item": i} for i in range(20)],
|
|
},
|
|
"nodes_summary": {
|
|
"total": 2,
|
|
"ready": 1,
|
|
"not_ready": 1,
|
|
"not_ready_names": ["titan-06"],
|
|
},
|
|
"flux": {
|
|
"items": [
|
|
{"name": str(i), "ready": False if i < 10 else None}
|
|
for i in range(20)
|
|
]
|
|
},
|
|
"pod_issues": {
|
|
"items": [
|
|
{
|
|
"namespace": "apps",
|
|
"pod": str(i),
|
|
"phase": "Pending",
|
|
"age_hours": 1,
|
|
}
|
|
for i in range(20)
|
|
],
|
|
"pending_oldest": [{"pod": "p"}],
|
|
},
|
|
"jobs": {
|
|
"failing": [{"job": "j", "age_hours": 1}],
|
|
"active_oldest": [{"job": "old"}],
|
|
},
|
|
"events": {"warnings_recent": [{"message": "warn"}]},
|
|
}
|
|
|
|
evidence = testing_triage._cluster_evidence(snapshot) # noqa: SLF001
|
|
|
|
assert len(evidence["health_bullets"]) == testing_triage._MAX_EVIDENCE_ITEMS # noqa: SLF001
|
|
assert evidence["nodes"]["not_ready_names"] == ["titan-06"]
|
|
assert evidence["jobs_failing"] == [{"job": "j", "age_hours": 1}]
|
|
|
|
|
|
def test_cluster_evidence_separates_active_incidents_from_context() -> None:
|
|
snapshot = {
|
|
"flux": {
|
|
"items": [
|
|
{"name": "ready-unknown", "ready": None},
|
|
{"name": "broken", "ready": False},
|
|
]
|
|
},
|
|
"pod_issues": {
|
|
"items": [
|
|
{
|
|
"namespace": "apps",
|
|
"pod": "old-restarts",
|
|
"phase": "Running",
|
|
"restarts": 100,
|
|
},
|
|
{
|
|
"namespace": "apps",
|
|
"pod": "starting",
|
|
"phase": "Pending",
|
|
"age_hours": 0.1,
|
|
},
|
|
{
|
|
"namespace": "apps",
|
|
"pod": "stuck",
|
|
"phase": "Pending",
|
|
"age_hours": 1,
|
|
},
|
|
{
|
|
"namespace": "veles",
|
|
"pod": "legacy",
|
|
"phase": "Pending",
|
|
"age_hours": 100,
|
|
},
|
|
]
|
|
},
|
|
"jobs": {
|
|
"failing": [
|
|
{"job": "recent", "age_hours": 1},
|
|
{"job": "historical", "age_hours": 100},
|
|
]
|
|
},
|
|
}
|
|
|
|
evidence = testing_triage._cluster_evidence(snapshot) # noqa: SLF001
|
|
|
|
assert evidence["flux_not_ready"] == [{"name": "broken", "ready": False}]
|
|
assert evidence["flux_in_progress"] == [
|
|
{"name": "ready-unknown", "ready": None}
|
|
]
|
|
assert [item["pod"] for item in evidence["pod_issues"]] == ["stuck"]
|
|
assert [item["pod"] for item in evidence["migration_residue"]] == ["legacy"]
|
|
assert evidence["jobs_failing"] == [{"job": "recent", "age_hours": 1}]
|
|
|
|
|
|
def test_vm_items_handles_success_failure_and_bad_values(monkeypatch) -> None:
|
|
class FakeResponse:
|
|
def __init__(self, payload) -> None: # type: ignore[no-untyped-def]
|
|
self.payload = payload
|
|
|
|
def raise_for_status(self) -> None:
|
|
return None
|
|
|
|
def json(self): # type: ignore[no-untyped-def]
|
|
return self.payload
|
|
|
|
class FakeClient:
|
|
def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def]
|
|
self.payload = kwargs.pop("payload", None)
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def get(self, url, params=None): # type: ignore[no-untyped-def]
|
|
assert url.endswith("/api/v1/query")
|
|
assert params == {"query": "up"}
|
|
return FakeResponse(
|
|
{
|
|
"status": "success",
|
|
"data": {
|
|
"result": [
|
|
{
|
|
"metric": {"suite": "ariadne", "__name__": "ignored"},
|
|
"value": [1, "2.5"],
|
|
},
|
|
{"metric": {"suite": "metis"}, "value": [1, "bad"]},
|
|
]
|
|
},
|
|
}
|
|
)
|
|
|
|
monkeypatch.setattr(testing_triage, "settings", SettingsStub(vm_url="http://vm"))
|
|
monkeypatch.setattr(testing_triage.httpx, "Client", FakeClient)
|
|
errors: list[str] = []
|
|
|
|
items = testing_triage._vm_items("up", errors) # noqa: SLF001
|
|
|
|
assert items == [
|
|
{"labels": {"suite": "ariadne"}, "value": 2.5},
|
|
{"labels": {"suite": "metis"}, "value": 0.0},
|
|
]
|
|
assert errors == []
|
|
|
|
|
|
def test_vm_items_records_errors(monkeypatch) -> None:
|
|
class BrokenClient:
|
|
def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def __enter__(self):
|
|
raise RuntimeError("network down")
|
|
|
|
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
monkeypatch.setattr(testing_triage, "settings", SettingsStub(vm_url="http://vm"))
|
|
monkeypatch.setattr(testing_triage.httpx, "Client", BrokenClient)
|
|
errors: list[str] = []
|
|
|
|
assert testing_triage._vm_items("up", errors) == [] # noqa: SLF001
|
|
assert errors == ["victoria_metrics: network down"]
|
|
|
|
|
|
def test_vm_items_handles_disabled_and_query_failure(monkeypatch) -> None:
|
|
class FailedResponse:
|
|
def raise_for_status(self) -> None:
|
|
return None
|
|
|
|
def json(self): # type: ignore[no-untyped-def]
|
|
return {"status": "error"}
|
|
|
|
class FailedClient:
|
|
def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def get(self, url, params=None): # type: ignore[no-untyped-def]
|
|
return FailedResponse()
|
|
|
|
errors: list[str] = []
|
|
monkeypatch.setattr(testing_triage, "settings", SettingsStub(vm_url=""))
|
|
assert testing_triage._vm_items("up", errors) == [] # noqa: SLF001
|
|
assert errors == []
|
|
|
|
monkeypatch.setattr(testing_triage, "settings", SettingsStub(vm_url="http://vm"))
|
|
monkeypatch.setattr(testing_triage.httpx, "Client", FailedClient)
|
|
assert testing_triage._vm_items("up", errors) == [] # noqa: SLF001
|
|
assert errors == ["victoria_metrics: query failed"]
|
|
|
|
|
|
def test_summary_and_markdown_helpers() -> None:
|
|
quality = {
|
|
"failed_runs_24h": {"items": [{"labels": {"suite": "ariadne"}, "value": 2}]},
|
|
"empty": {"items": []},
|
|
"bad": "skip",
|
|
}
|
|
cluster = {
|
|
"flux_not_ready": [{"name": "monitoring"}],
|
|
"pod_issues": [],
|
|
"jobs_failing": [{"job": "job"}],
|
|
"collected_at": "now",
|
|
}
|
|
jenkins = {"failed_builds": [{"job": "bstein-dev-home"}]}
|
|
|
|
summary = testing_triage._summary(cluster, quality, jenkins, []) # noqa: SLF001
|
|
markdown = testing_triage._render_markdown( # noqa: SLF001
|
|
{
|
|
"generated_at": "now",
|
|
"summary": summary,
|
|
"evidence": {"cluster": cluster, "quality": quality, "jenkins": jenkins},
|
|
"unknowns": ["missing vm"],
|
|
}
|
|
)
|
|
|
|
assert summary["status"] == "needs_attention"
|
|
assert summary["failed_suites"] == ["ariadne", "bstein_home"]
|
|
assert "- Flux: monitoring" in markdown
|
|
assert "- failed_runs_24h: {'suite': 'ariadne'} value=2" in markdown
|
|
assert "- missing vm" in markdown
|
|
assert testing_triage._markdown_named_items("Pods", ["bad"], "pod") == [
|
|
"- Pods: none"
|
|
] # noqa: SLF001
|