498 lines
19 KiB
Python
498 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from .core_a import _BYTES_GB, _BYTES_KB, _BYTES_MB
|
|
from .core_b import *
|
|
|
|
|
|
def _format_float(value: Any) -> str:
|
|
try:
|
|
numeric = float(value)
|
|
except (TypeError, ValueError):
|
|
return str(value)
|
|
return f"{numeric:.2f}".rstrip("0").rstrip(".")
|
|
|
|
|
|
def _format_rate_bytes(value: Any) -> str:
|
|
try:
|
|
numeric = float(value)
|
|
except (TypeError, ValueError):
|
|
return str(value)
|
|
if numeric >= _BYTES_MB:
|
|
return f"{numeric / _BYTES_MB:.2f} MB/s"
|
|
if numeric >= _BYTES_KB:
|
|
return f"{numeric / _BYTES_KB:.2f} KB/s"
|
|
return f"{numeric:.2f} B/s"
|
|
|
|
|
|
def _format_bytes(value: Any) -> str:
|
|
try:
|
|
numeric = float(value)
|
|
except (TypeError, ValueError):
|
|
return str(value)
|
|
if numeric >= _BYTES_GB:
|
|
return f"{numeric / _BYTES_GB:.2f} GB"
|
|
if numeric >= _BYTES_MB:
|
|
return f"{numeric / _BYTES_MB:.2f} MB"
|
|
if numeric >= _BYTES_KB:
|
|
return f"{numeric / _BYTES_KB:.2f} KB"
|
|
return f"{numeric:.2f} B"
|
|
|
|
|
|
def _format_kv_map(values: dict[str, Any]) -> str:
|
|
parts = []
|
|
for key, value in values.items():
|
|
parts.append(f"{key}={value}")
|
|
return ", ".join(parts)
|
|
|
|
|
|
def _format_names(names: list[str]) -> str:
|
|
if not names:
|
|
return ""
|
|
return ", ".join(sorted(names))
|
|
|
|
|
|
def _append_nodes(lines: list[str], summary: dict[str, Any]) -> None: # noqa: C901
|
|
nodes = summary.get("nodes") if isinstance(summary.get("nodes"), dict) else {}
|
|
if not nodes:
|
|
return
|
|
workers = {}
|
|
if isinstance(summary.get("nodes_summary"), dict):
|
|
workers = summary["nodes_summary"].get("workers") or {}
|
|
workers_total = workers.get("total")
|
|
workers_ready = workers.get("ready")
|
|
workers_str = ""
|
|
if workers_total is not None and workers_ready is not None:
|
|
workers_str = f", workers_ready={workers_ready}/{workers_total}"
|
|
total = nodes.get("total")
|
|
ready = nodes.get("ready")
|
|
not_ready = nodes.get("not_ready")
|
|
if not_ready is None:
|
|
not_ready = 0
|
|
lines.append(f"nodes: total={total}, ready={ready}, not_ready={not_ready}{workers_str}")
|
|
if total is not None:
|
|
lines.append(f"nodes_total: {total}")
|
|
if ready is not None:
|
|
lines.append(f"nodes_ready: {ready}")
|
|
if not_ready is not None:
|
|
lines.append(f"nodes_not_ready_count: {not_ready}")
|
|
if not isinstance(summary.get("nodes_summary"), dict):
|
|
return
|
|
not_ready_names = summary["nodes_summary"].get("not_ready_names") or []
|
|
if not_ready_names:
|
|
lines.append("nodes_not_ready: " + _format_names(not_ready_names))
|
|
by_arch = summary["nodes_summary"].get("by_arch") or {}
|
|
if isinstance(by_arch, dict) and by_arch:
|
|
lines.append("archs: " + _format_kv_map(by_arch))
|
|
by_role = summary["nodes_summary"].get("by_role") or {}
|
|
if isinstance(by_role, dict) and by_role:
|
|
lines.append("roles: " + _format_kv_map(by_role))
|
|
|
|
|
|
def _append_hardware(lines: list[str], summary: dict[str, Any]) -> None:
|
|
hardware = summary.get("hardware") if isinstance(summary.get("hardware"), dict) else {}
|
|
if not hardware:
|
|
return
|
|
parts = []
|
|
for key, names in hardware.items():
|
|
if not isinstance(names, list):
|
|
continue
|
|
label = f"{key}={len(names)}"
|
|
name_list = _format_names([str(name) for name in names if name])
|
|
if name_list:
|
|
label = f"{label} ({name_list})"
|
|
parts.append(label)
|
|
if parts:
|
|
lines.append("hardware: " + "; ".join(sorted(parts)))
|
|
|
|
|
|
def _append_hardware_groups(lines: list[str], summary: dict[str, Any]) -> None:
|
|
hardware = summary.get("hardware") if isinstance(summary.get("hardware"), dict) else {}
|
|
if not hardware:
|
|
return
|
|
parts = []
|
|
for key, names in hardware.items():
|
|
if not isinstance(names, list):
|
|
continue
|
|
name_list = _format_names([str(name) for name in names if name])
|
|
if name_list:
|
|
parts.append(f"{key}={name_list}")
|
|
if parts:
|
|
lines.append("hardware_nodes: " + "; ".join(sorted(parts)))
|
|
|
|
|
|
def _append_node_ages(lines: list[str], summary: dict[str, Any]) -> None:
|
|
ages = summary.get("node_ages") if isinstance(summary.get("node_ages"), list) else []
|
|
if not ages:
|
|
return
|
|
parts = []
|
|
for entry in ages[:3]:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
name = entry.get("name")
|
|
age = entry.get("age_hours")
|
|
if name and isinstance(age, (int, float)):
|
|
parts.append(f"{name}={_format_float(age)}h")
|
|
if parts:
|
|
lines.append("node_age_top: " + "; ".join(parts))
|
|
|
|
|
|
def _append_node_taints(lines: list[str], summary: dict[str, Any]) -> None:
|
|
taints = summary.get("node_taints") if isinstance(summary.get("node_taints"), dict) else {}
|
|
if not taints:
|
|
return
|
|
parts = []
|
|
for key, names in taints.items():
|
|
if not isinstance(names, list):
|
|
continue
|
|
name_list = _format_names([str(name) for name in names if name])
|
|
parts.append(f"{key}={len(names)} ({name_list})" if name_list else f"{key}={len(names)}")
|
|
if parts:
|
|
lines.append("node_taints: " + "; ".join(sorted(parts)))
|
|
|
|
|
|
def _append_node_facts(lines: list[str], summary: dict[str, Any]) -> None:
|
|
def top_counts(label: str, counts: dict[str, int], limit: int = 4) -> None:
|
|
if not counts:
|
|
return
|
|
top = sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:limit]
|
|
rendered = "; ".join([f"{name}={count}" for name, count in top])
|
|
if rendered:
|
|
lines.append(f"{label}: {rendered}")
|
|
|
|
top_counts("node_arch", summary.get("node_arch_counts") or {})
|
|
top_counts("node_os", summary.get("node_os_counts") or {})
|
|
top_counts("node_kubelet_versions", summary.get("node_kubelet_versions") or {})
|
|
top_counts("node_kernel_versions", summary.get("node_kernel_versions") or {})
|
|
top_counts("node_runtime_versions", summary.get("node_runtime_versions") or {})
|
|
top_counts("node_roles", summary.get("node_role_counts") or {})
|
|
|
|
|
|
def _append_pressure(lines: list[str], summary: dict[str, Any]) -> None:
|
|
pressure = summary.get("pressure_nodes")
|
|
if not isinstance(pressure, dict) or not pressure:
|
|
return
|
|
parts = []
|
|
for cond, nodes in sorted(pressure.items()):
|
|
if not nodes:
|
|
continue
|
|
name_list = _format_names([str(name) for name in nodes if name])
|
|
parts.append(f"{cond}={len(nodes)} ({name_list})" if name_list else f"{cond}={len(nodes)}")
|
|
if parts:
|
|
lines.append("node_pressure: " + "; ".join(parts))
|
|
|
|
|
|
def _append_pods(lines: list[str], summary: dict[str, Any]) -> None:
|
|
pods = summary.get("pods") if isinstance(summary.get("pods"), dict) else {}
|
|
if not pods:
|
|
return
|
|
lines.append(
|
|
"pods: running={running}, pending={pending}, failed={failed}, succeeded={succeeded}".format(
|
|
running=pods.get("running"),
|
|
pending=pods.get("pending"),
|
|
failed=pods.get("failed"),
|
|
succeeded=pods.get("succeeded"),
|
|
)
|
|
)
|
|
|
|
|
|
def _append_capacity(lines: list[str], summary: dict[str, Any]) -> None:
|
|
capacity = summary.get("capacity") if isinstance(summary.get("capacity"), dict) else {}
|
|
if not capacity:
|
|
return
|
|
parts = []
|
|
if capacity.get("cpu") is not None:
|
|
parts.append(f"cpu={_format_float(capacity.get('cpu'))}")
|
|
if capacity.get("allocatable_cpu") is not None:
|
|
parts.append(f"alloc_cpu={_format_float(capacity.get('allocatable_cpu'))}")
|
|
if capacity.get("mem_bytes") is not None:
|
|
parts.append(f"mem={_format_bytes(capacity.get('mem_bytes'))}")
|
|
if capacity.get("allocatable_mem_bytes") is not None:
|
|
parts.append(f"alloc_mem={_format_bytes(capacity.get('allocatable_mem_bytes'))}")
|
|
if capacity.get("pods") is not None:
|
|
parts.append(f"pods={_format_float(capacity.get('pods'))}")
|
|
if capacity.get("allocatable_pods") is not None:
|
|
parts.append(f"alloc_pods={_format_float(capacity.get('allocatable_pods'))}")
|
|
if parts:
|
|
lines.append("capacity: " + "; ".join(parts))
|
|
|
|
|
|
def _append_namespace_pods(lines: list[str], summary: dict[str, Any]) -> None:
|
|
namespaces = summary.get("namespace_pods")
|
|
if not isinstance(namespaces, list) or not namespaces:
|
|
return
|
|
top = sorted(
|
|
(item for item in namespaces if isinstance(item, dict)),
|
|
key=lambda item: (-int(item.get("pods_total") or 0), item.get("namespace") or ""),
|
|
)[:8]
|
|
parts = []
|
|
for item in top:
|
|
name = item.get("namespace")
|
|
total = item.get("pods_total")
|
|
running = item.get("pods_running")
|
|
if not name:
|
|
continue
|
|
label = f"{name}={total}"
|
|
if running is not None:
|
|
label = f"{label} (running={running})"
|
|
parts.append(label)
|
|
if parts:
|
|
lines.append("namespaces_top: " + "; ".join(parts))
|
|
|
|
|
|
def _append_namespace_nodes(lines: list[str], summary: dict[str, Any]) -> None:
|
|
namespace_nodes = summary.get("namespace_nodes")
|
|
if not isinstance(namespace_nodes, list) or not namespace_nodes:
|
|
return
|
|
top = sorted(
|
|
(item for item in namespace_nodes if isinstance(item, dict)),
|
|
key=lambda item: (-int(item.get("pods_total") or 0), item.get("namespace") or ""),
|
|
)[:8]
|
|
parts = []
|
|
for item in top:
|
|
namespace = item.get("namespace")
|
|
pods_total = item.get("pods_total")
|
|
primary = item.get("primary_node")
|
|
if namespace:
|
|
label = f"{namespace}={pods_total}"
|
|
if primary:
|
|
label = f"{label} (primary={primary})"
|
|
parts.append(label)
|
|
if parts:
|
|
lines.append("namespace_nodes_top: " + "; ".join(parts))
|
|
|
|
|
|
def _append_node_pods(lines: list[str], summary: dict[str, Any]) -> None: # noqa: C901
|
|
node_pods = summary.get("node_pods")
|
|
if not isinstance(node_pods, list) or not node_pods:
|
|
return
|
|
sortable: list[dict[str, Any]] = []
|
|
for item in node_pods:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
try:
|
|
pods_value = int(item.get("pods_total") or 0)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
sortable.append({**item, "pods_total": pods_value})
|
|
top = sorted(sortable, key=lambda item: (-int(item.get("pods_total") or 0), item.get("node") or ""))[:8]
|
|
max_entry = None
|
|
for entry in node_pods:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
pods_total = entry.get("pods_total")
|
|
try:
|
|
pods_value = int(pods_total)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if max_entry is None or pods_value > max_entry["pods_total"]:
|
|
max_entry = {
|
|
"node": entry.get("node"),
|
|
"pods_total": pods_value,
|
|
"namespaces_top": entry.get("namespaces_top") or [],
|
|
}
|
|
parts = []
|
|
for item in top:
|
|
node = item.get("node")
|
|
pods_total = item.get("pods_total")
|
|
namespaces = item.get("namespaces_top") or []
|
|
ns_label = ""
|
|
if namespaces:
|
|
ns_label = ", ".join([f"{name}={count}" for name, count in namespaces])
|
|
if node:
|
|
label = f"{node}={pods_total}"
|
|
if ns_label:
|
|
label = f"{label} ({ns_label})"
|
|
parts.append(label)
|
|
if parts:
|
|
lines.append("node_pods_top: " + "; ".join(parts))
|
|
if max_entry and isinstance(max_entry.get("node"), str):
|
|
ns_label = ""
|
|
namespaces = max_entry.get("namespaces_top") or []
|
|
if namespaces:
|
|
ns_label = ", ".join([f"{name}={count}" for name, count in namespaces])
|
|
label = f"{max_entry.get('node')}={max_entry.get('pods_total')}"
|
|
if ns_label:
|
|
label = f"{label} ({ns_label})"
|
|
lines.append("node_pods_max: " + label)
|
|
for item in top:
|
|
node = item.get("node")
|
|
namespaces = item.get("namespaces_top") or []
|
|
if not node or not namespaces:
|
|
continue
|
|
ns_label = ", ".join([f"{name}={count}" for name, count in namespaces])
|
|
lines.append(f"node_namespaces_top: {node} ({ns_label})")
|
|
|
|
|
|
def _append_pod_issues(lines: list[str], summary: dict[str, Any]) -> None:
|
|
pod_issues = summary.get("pod_issues") if isinstance(summary.get("pod_issues"), dict) else {}
|
|
if not pod_issues:
|
|
return
|
|
counts_line = _format_pod_issue_counts(pod_issues)
|
|
if counts_line:
|
|
lines.append(counts_line)
|
|
top_line = _format_pod_issue_top(pod_issues)
|
|
if top_line:
|
|
lines.append(top_line)
|
|
pending_line = _format_pod_pending_oldest(pod_issues)
|
|
if pending_line:
|
|
lines.append(pending_line)
|
|
pending_over_line = _format_pod_pending_over_15m(pod_issues)
|
|
if pending_over_line:
|
|
lines.append(pending_over_line)
|
|
reasons_line = _format_pod_waiting_reasons(pod_issues)
|
|
if reasons_line:
|
|
lines.append(reasons_line)
|
|
|
|
|
|
def _format_pod_issue_counts(pod_issues: dict[str, Any]) -> str:
|
|
counts = pod_issues.get("counts") if isinstance(pod_issues.get("counts"), dict) else {}
|
|
if not counts:
|
|
return ""
|
|
parts = []
|
|
for key in ("Failed", "Pending", "Unknown"):
|
|
if key in counts:
|
|
parts.append(f"{key}={counts.get(key)}")
|
|
return "pod_issues: " + "; ".join(parts) if parts else ""
|
|
|
|
|
|
def _format_pod_issue_top(pod_issues: dict[str, Any]) -> str:
|
|
items = pod_issues.get("items") if isinstance(pod_issues.get("items"), list) else []
|
|
if not items:
|
|
return ""
|
|
top = []
|
|
for item in items[:5]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
namespace = item.get("namespace")
|
|
pod = item.get("pod")
|
|
if not namespace or not pod:
|
|
continue
|
|
phase = item.get("phase") or ""
|
|
restarts = item.get("restarts") or 0
|
|
top.append(f"{namespace}/{pod}({phase},r={restarts})")
|
|
return "pod_issues_top: " + "; ".join(top) if top else ""
|
|
|
|
|
|
def _format_pod_pending_oldest(pod_issues: dict[str, Any]) -> str:
|
|
pending = pod_issues.get("pending_oldest") if isinstance(pod_issues.get("pending_oldest"), list) else []
|
|
if not pending:
|
|
return ""
|
|
parts = []
|
|
for item in pending[:5]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
namespace = item.get("namespace")
|
|
pod = item.get("pod")
|
|
age = item.get("age_hours")
|
|
reason = item.get("reason") or ""
|
|
if namespace and pod and age is not None:
|
|
label = f"{namespace}/{pod}={_format_float(age)}h"
|
|
if reason:
|
|
label = f"{label} ({reason})"
|
|
parts.append(label)
|
|
return "pods_pending_oldest: " + "; ".join(parts) if parts else ""
|
|
|
|
|
|
def _format_pod_waiting_reasons(pod_issues: dict[str, Any]) -> str:
|
|
reasons = pod_issues.get("waiting_reasons") if isinstance(pod_issues.get("waiting_reasons"), dict) else {}
|
|
if not reasons:
|
|
return ""
|
|
pairs = sorted(reasons.items(), key=lambda item: (-item[1], item[0]))[:5]
|
|
return "pod_waiting_reasons: " + "; ".join([f"{key}={val}" for key, val in pairs])
|
|
|
|
|
|
def _format_pod_pending_over_15m(pod_issues: dict[str, Any]) -> str:
|
|
count = pod_issues.get("pending_over_15m")
|
|
if count is None:
|
|
return ""
|
|
try:
|
|
count_val = int(count)
|
|
except (TypeError, ValueError):
|
|
return ""
|
|
return f"pods_pending_over_15m: {count_val}"
|
|
|
|
|
|
def _append_workload_health(lines: list[str], summary: dict[str, Any]) -> None:
|
|
health = summary.get("workloads_health") if isinstance(summary.get("workloads_health"), dict) else {}
|
|
if not health:
|
|
return
|
|
deployments = health.get("deployments") if isinstance(health.get("deployments"), dict) else {}
|
|
statefulsets = health.get("statefulsets") if isinstance(health.get("statefulsets"), dict) else {}
|
|
daemonsets = health.get("daemonsets") if isinstance(health.get("daemonsets"), dict) else {}
|
|
total_not_ready = 0
|
|
for entry in (deployments, statefulsets, daemonsets):
|
|
total_not_ready += int(entry.get("not_ready") or 0)
|
|
lines.append(
|
|
"workloads_not_ready: "
|
|
f"deployments={deployments.get('not_ready', 0)}, "
|
|
f"statefulsets={statefulsets.get('not_ready', 0)}, "
|
|
f"daemonsets={daemonsets.get('not_ready', 0)} "
|
|
f"(total={total_not_ready})"
|
|
)
|
|
|
|
|
|
def _append_node_usage_stats(lines: list[str], summary: dict[str, Any]) -> None:
|
|
metrics = summary.get("metrics") if isinstance(summary.get("metrics"), dict) else {}
|
|
stats = metrics.get("node_usage_stats") if isinstance(metrics.get("node_usage_stats"), dict) else {}
|
|
if not stats:
|
|
return
|
|
parts = []
|
|
for key in ("cpu", "ram", "net", "io", "disk"):
|
|
entry = stats.get(key) if isinstance(stats.get(key), dict) else {}
|
|
avg = entry.get("avg")
|
|
if avg is None:
|
|
continue
|
|
value = _format_rate_bytes(avg) if key in {"net", "io"} else _format_float(avg)
|
|
parts.append(f"{key}={value}")
|
|
if parts:
|
|
lines.append("node_usage_avg: " + "; ".join(parts))
|
|
|
|
|
|
def _append_events(lines: list[str], summary: dict[str, Any]) -> None:
|
|
events = summary.get("events") if isinstance(summary.get("events"), dict) else {}
|
|
if not events:
|
|
return
|
|
total = events.get("warnings_total")
|
|
by_reason = events.get("warnings_by_reason") if isinstance(events.get("warnings_by_reason"), dict) else {}
|
|
if total is None:
|
|
return
|
|
if by_reason:
|
|
top = sorted(by_reason.items(), key=lambda item: (-item[1], item[0]))[:3]
|
|
reasons = "; ".join([f"{reason}={count}" for reason, count in top])
|
|
lines.append(f"warnings: total={total}; top={reasons}")
|
|
else:
|
|
lines.append(f"warnings: total={total}")
|
|
def _append_pvc_usage(lines: list[str], summary: dict[str, Any]) -> None:
|
|
pvc_usage = summary.get("pvc_usage_top")
|
|
if not isinstance(pvc_usage, list) or not pvc_usage:
|
|
return
|
|
parts = []
|
|
for entry in pvc_usage:
|
|
metric = entry.get("metric") if isinstance(entry, dict) else {}
|
|
namespace = metric.get("namespace")
|
|
pvc = metric.get("persistentvolumeclaim")
|
|
value = entry.get("value")
|
|
if namespace and pvc:
|
|
parts.append(f"{namespace}/{pvc}={_format_float(value)}%")
|
|
if parts:
|
|
lines.append("pvc_usage_top: " + "; ".join(parts))
|
|
|
|
|
|
def _append_root_disk_headroom(lines: list[str], summary: dict[str, Any]) -> None:
|
|
headroom = summary.get("root_disk_low_headroom")
|
|
if not isinstance(headroom, list) or not headroom:
|
|
return
|
|
parts = []
|
|
for entry in headroom:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
node = entry.get("node")
|
|
headroom_pct = entry.get("headroom_pct")
|
|
if node and headroom_pct is not None:
|
|
parts.append(f"{node}={_format_float(headroom_pct)}%")
|
|
if parts:
|
|
lines.append("root_disk_low_headroom: " + "; ".join(parts))
|
|
__all__ = [name for name in globals() if not name.startswith("__")]
|