320 lines
12 KiB
Python
320 lines
12 KiB
Python
from __future__ import annotations
|
|
from .common import *
|
|
from .nodes import *
|
|
from .k8s import *
|
|
|
|
def _node_pods_top(node_pods: list[dict[str, Any]], limit: int = 5) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
for entry in node_pods[:limit]:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
output.append(
|
|
{
|
|
"node": entry.get("node"),
|
|
"pods_total": entry.get("pods_total"),
|
|
"pods_running": entry.get("pods_running"),
|
|
"namespaces_top": entry.get("namespaces_top") or [],
|
|
}
|
|
)
|
|
return output
|
|
|
|
|
|
def _record_pending_pod(
|
|
pending_oldest: list[dict[str, Any]],
|
|
info: dict[str, Any],
|
|
) -> bool:
|
|
age_hours = info.get("age_hours")
|
|
if age_hours is None:
|
|
return False
|
|
pending_oldest.append(info)
|
|
return age_hours >= _PENDING_15M_HOURS
|
|
|
|
|
|
def _update_pod_issue(
|
|
pod: dict[str, Any],
|
|
acc: dict[str, Any],
|
|
) -> None:
|
|
metadata = pod.get("metadata") if isinstance(pod.get("metadata"), dict) else {}
|
|
status = pod.get("status") if isinstance(pod.get("status"), dict) else {}
|
|
spec = pod.get("spec") if isinstance(pod.get("spec"), dict) else {}
|
|
namespace = metadata.get("namespace") if isinstance(metadata.get("namespace"), str) else ""
|
|
name = metadata.get("name") if isinstance(metadata.get("name"), str) else ""
|
|
created_at = (
|
|
metadata.get("creationTimestamp")
|
|
if isinstance(metadata.get("creationTimestamp"), str)
|
|
else ""
|
|
)
|
|
age_hours = _age_hours(created_at)
|
|
if not name or not namespace:
|
|
return
|
|
phase = status.get("phase") if isinstance(status.get("phase"), str) else ""
|
|
restarts = 0
|
|
waiting_reasons: list[str] = []
|
|
for container in status.get("containerStatuses") or []:
|
|
if not isinstance(container, dict):
|
|
continue
|
|
restarts += int(container.get("restartCount") or 0)
|
|
state = container.get("state") if isinstance(container.get("state"), dict) else {}
|
|
waiting = state.get("waiting") if isinstance(state.get("waiting"), dict) else {}
|
|
reason = waiting.get("reason")
|
|
if isinstance(reason, str) and reason:
|
|
waiting_reasons.append(reason)
|
|
acc["waiting_reasons"][reason] = acc["waiting_reasons"].get(reason, 0) + 1
|
|
phase_reason = status.get("reason")
|
|
if isinstance(phase_reason, str) and phase_reason:
|
|
acc["phase_reasons"][phase_reason] = acc["phase_reasons"].get(phase_reason, 0) + 1
|
|
if phase in acc["counts"]:
|
|
acc["counts"][phase] += 1
|
|
if phase in _PHASE_SEVERITY or restarts > 0:
|
|
acc["items"].append(
|
|
{
|
|
"namespace": namespace,
|
|
"pod": name,
|
|
"node": spec.get("nodeName") or "",
|
|
"phase": phase,
|
|
"reason": status.get("reason") or "",
|
|
"restarts": restarts,
|
|
"waiting_reasons": sorted(set(waiting_reasons)),
|
|
"created_at": created_at,
|
|
"age_hours": age_hours,
|
|
}
|
|
)
|
|
if phase == "Pending":
|
|
info = {
|
|
"namespace": namespace,
|
|
"pod": name,
|
|
"node": spec.get("nodeName") or "",
|
|
"age_hours": age_hours,
|
|
"reason": status.get("reason") or "",
|
|
}
|
|
if _record_pending_pod(acc["pending_oldest"], info):
|
|
acc["pending_over_15m"] += 1
|
|
|
|
|
|
def _summarize_pod_issues(payload: dict[str, Any]) -> dict[str, Any]:
|
|
acc = {
|
|
"items": [],
|
|
"counts": {key: 0 for key in _PHASE_SEVERITY},
|
|
"pending_oldest": [],
|
|
"pending_over_15m": 0,
|
|
"waiting_reasons": {},
|
|
"phase_reasons": {},
|
|
}
|
|
for pod in _items(payload):
|
|
if isinstance(pod, dict):
|
|
_update_pod_issue(pod, acc)
|
|
items = acc["items"]
|
|
items.sort(
|
|
key=lambda item: (
|
|
-_PHASE_SEVERITY.get(item.get("phase") or "", 0),
|
|
-(item.get("restarts") or 0),
|
|
item.get("namespace") or "",
|
|
item.get("pod") or "",
|
|
)
|
|
)
|
|
pending_oldest = acc["pending_oldest"]
|
|
pending_oldest.sort(key=lambda item: -(item.get("age_hours") or 0.0))
|
|
return {
|
|
"counts": acc["counts"],
|
|
"items": items[:20],
|
|
"pending_oldest": pending_oldest[:10],
|
|
"pending_over_15m": acc["pending_over_15m"],
|
|
"waiting_reasons": acc["waiting_reasons"],
|
|
"phase_reasons": acc["phase_reasons"],
|
|
}
|
|
|
|
|
|
def _summarize_jobs(payload: dict[str, Any]) -> dict[str, Any]:
|
|
totals = {"total": 0, "active": 0, "failed": 0, "succeeded": 0}
|
|
by_namespace: dict[str, dict[str, int]] = {}
|
|
failing: list[dict[str, Any]] = []
|
|
active_oldest: list[dict[str, Any]] = []
|
|
for job in _items(payload):
|
|
metadata = job.get("metadata") if isinstance(job.get("metadata"), dict) else {}
|
|
status = job.get("status") if isinstance(job.get("status"), dict) else {}
|
|
name = metadata.get("name") if isinstance(metadata.get("name"), str) else ""
|
|
namespace = metadata.get("namespace") if isinstance(metadata.get("namespace"), str) else ""
|
|
created_at = (
|
|
metadata.get("creationTimestamp")
|
|
if isinstance(metadata.get("creationTimestamp"), str)
|
|
else ""
|
|
)
|
|
if not name or not namespace:
|
|
continue
|
|
active = int(status.get("active") or 0)
|
|
failed = int(status.get("failed") or 0)
|
|
succeeded = int(status.get("succeeded") or 0)
|
|
totals["total"] += 1
|
|
totals["active"] += active
|
|
totals["failed"] += failed
|
|
totals["succeeded"] += succeeded
|
|
entry = by_namespace.setdefault(namespace, {"active": 0, "failed": 0, "succeeded": 0})
|
|
entry["active"] += active
|
|
entry["failed"] += failed
|
|
entry["succeeded"] += succeeded
|
|
age_hours = _age_hours(created_at)
|
|
if failed > 0:
|
|
failing.append(
|
|
{
|
|
"namespace": namespace,
|
|
"job": name,
|
|
"failed": failed,
|
|
"age_hours": age_hours,
|
|
}
|
|
)
|
|
if active > 0 and age_hours is not None:
|
|
active_oldest.append(
|
|
{
|
|
"namespace": namespace,
|
|
"job": name,
|
|
"active": active,
|
|
"age_hours": age_hours,
|
|
}
|
|
)
|
|
failing.sort(
|
|
key=lambda item: (
|
|
-(item.get("failed") or 0),
|
|
-(item.get("age_hours") or 0.0),
|
|
item.get("namespace") or "",
|
|
item.get("job") or "",
|
|
)
|
|
)
|
|
active_oldest.sort(key=lambda item: -(item.get("age_hours") or 0.0))
|
|
namespace_summary = [
|
|
{
|
|
"namespace": ns,
|
|
"active": stats.get("active", 0),
|
|
"failed": stats.get("failed", 0),
|
|
"succeeded": stats.get("succeeded", 0),
|
|
}
|
|
for ns, stats in by_namespace.items()
|
|
]
|
|
namespace_summary.sort(
|
|
key=lambda item: (
|
|
-(item.get("active") or 0),
|
|
-(item.get("failed") or 0),
|
|
item.get("namespace") or "",
|
|
)
|
|
)
|
|
return {
|
|
"totals": totals,
|
|
"by_namespace": namespace_summary[:20],
|
|
"failing": failing[:20],
|
|
"active_oldest": active_oldest[:20],
|
|
}
|
|
|
|
|
|
def _summarize_deployments(payload: dict[str, Any]) -> dict[str, Any]:
|
|
items = _items(payload)
|
|
unhealthy: list[dict[str, Any]] = []
|
|
for dep in items:
|
|
metadata = dep.get("metadata") if isinstance(dep.get("metadata"), dict) else {}
|
|
spec = dep.get("spec") if isinstance(dep.get("spec"), dict) else {}
|
|
status = dep.get("status") if isinstance(dep.get("status"), dict) else {}
|
|
name = metadata.get("name") if isinstance(metadata.get("name"), str) else ""
|
|
namespace = metadata.get("namespace") if isinstance(metadata.get("namespace"), str) else ""
|
|
desired = int(spec.get("replicas") or 0)
|
|
ready = int(status.get("readyReplicas") or 0)
|
|
available = int(status.get("availableReplicas") or 0)
|
|
updated = int(status.get("updatedReplicas") or 0)
|
|
if desired <= 0:
|
|
continue
|
|
if ready < desired or available < desired:
|
|
unhealthy.append(
|
|
{
|
|
"name": name,
|
|
"namespace": namespace,
|
|
"desired": desired,
|
|
"ready": ready,
|
|
"available": available,
|
|
"updated": updated,
|
|
}
|
|
)
|
|
unhealthy.sort(key=lambda item: (item.get("namespace") or "", item.get("name") or ""))
|
|
return {
|
|
"total": len(items),
|
|
"not_ready": len(unhealthy),
|
|
"items": unhealthy,
|
|
}
|
|
|
|
|
|
def _summarize_statefulsets(payload: dict[str, Any]) -> dict[str, Any]:
|
|
items = _items(payload)
|
|
unhealthy: list[dict[str, Any]] = []
|
|
for st in items:
|
|
metadata = st.get("metadata") if isinstance(st.get("metadata"), dict) else {}
|
|
spec = st.get("spec") if isinstance(st.get("spec"), dict) else {}
|
|
status = st.get("status") if isinstance(st.get("status"), dict) else {}
|
|
name = metadata.get("name") if isinstance(metadata.get("name"), str) else ""
|
|
namespace = metadata.get("namespace") if isinstance(metadata.get("namespace"), str) else ""
|
|
desired = int(spec.get("replicas") or 0)
|
|
ready = int(status.get("readyReplicas") or 0)
|
|
current = int(status.get("currentReplicas") or 0)
|
|
updated = int(status.get("updatedReplicas") or 0)
|
|
if desired <= 0:
|
|
continue
|
|
if ready < desired:
|
|
unhealthy.append(
|
|
{
|
|
"name": name,
|
|
"namespace": namespace,
|
|
"desired": desired,
|
|
"ready": ready,
|
|
"current": current,
|
|
"updated": updated,
|
|
}
|
|
)
|
|
unhealthy.sort(key=lambda item: (item.get("namespace") or "", item.get("name") or ""))
|
|
return {
|
|
"total": len(items),
|
|
"not_ready": len(unhealthy),
|
|
"items": unhealthy,
|
|
}
|
|
|
|
|
|
def _summarize_daemonsets(payload: dict[str, Any]) -> dict[str, Any]:
|
|
items = _items(payload)
|
|
unhealthy: list[dict[str, Any]] = []
|
|
for ds in items:
|
|
metadata = ds.get("metadata") if isinstance(ds.get("metadata"), dict) else {}
|
|
status = ds.get("status") if isinstance(ds.get("status"), dict) else {}
|
|
name = metadata.get("name") if isinstance(metadata.get("name"), str) else ""
|
|
namespace = metadata.get("namespace") if isinstance(metadata.get("namespace"), str) else ""
|
|
desired = int(status.get("desiredNumberScheduled") or 0)
|
|
ready = int(status.get("numberReady") or 0)
|
|
updated = int(status.get("updatedNumberScheduled") or 0)
|
|
if desired <= 0:
|
|
continue
|
|
if ready < desired:
|
|
unhealthy.append(
|
|
{
|
|
"name": name,
|
|
"namespace": namespace,
|
|
"desired": desired,
|
|
"ready": ready,
|
|
"updated": updated,
|
|
}
|
|
)
|
|
unhealthy.sort(key=lambda item: (item.get("namespace") or "", item.get("name") or ""))
|
|
return {
|
|
"total": len(items),
|
|
"not_ready": len(unhealthy),
|
|
"items": unhealthy,
|
|
}
|
|
|
|
|
|
def _summarize_workload_health(
|
|
deployments: dict[str, Any],
|
|
statefulsets: dict[str, Any],
|
|
daemonsets: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"deployments": deployments,
|
|
"statefulsets": statefulsets,
|
|
"daemonsets": daemonsets,
|
|
}
|
|
|
|
|
|
|
|
__all__ = [name for name in globals() if not name.startswith("__")]
|