310 lines
12 KiB
Python
310 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
from typing import Any, Callable
|
|
|
|
import httpx
|
|
|
|
from ..settings import settings
|
|
from .cluster_state_contract import *
|
|
from .cluster_state_flux_events import *
|
|
from .cluster_state_relationships import *
|
|
|
|
|
|
def _facade_override(name: str, original: Callable[..., Any]) -> Callable[..., Any] | None:
|
|
facade = sys.modules.get("ariadne.services.cluster_state")
|
|
candidate = getattr(facade, name, None) if facade is not None else None
|
|
if candidate is not None and candidate is not original:
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def _vm_query(expr: str) -> list[dict[str, Any]] | None:
|
|
base = settings.vm_url
|
|
if not base:
|
|
return None
|
|
url = f"{base.rstrip('/')}/api/v1/query"
|
|
params = {"query": expr}
|
|
with httpx.Client(timeout=settings.cluster_state_vm_timeout_sec) as client:
|
|
resp = client.get(url, params=params)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
if payload.get("status") != "success":
|
|
return None
|
|
data = payload.get("data") if isinstance(payload.get("data"), dict) else {}
|
|
result = data.get("result")
|
|
return result if isinstance(result, list) else None
|
|
|
|
|
|
def _vm_scalar(expr: str) -> float | None:
|
|
override = _facade_override("_vm_scalar", _vm_scalar)
|
|
if override is not None:
|
|
return override(expr)
|
|
result = _vm_query(expr)
|
|
if not result:
|
|
return None
|
|
value = result[0].get("value") if isinstance(result[0], dict) else None
|
|
if not isinstance(value, list) or len(value) < _VALUE_PAIR_LEN:
|
|
return None
|
|
try:
|
|
return float(value[1])
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _vm_vector(expr: str) -> list[dict[str, Any]]:
|
|
override = _facade_override("_vm_vector", _vm_vector)
|
|
if override is not None:
|
|
return override(expr)
|
|
result = _vm_query(expr) or []
|
|
output: list[dict[str, Any]] = []
|
|
for item in result:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
value = item.get("value") if isinstance(item.get("value"), list) else []
|
|
if len(value) < _VALUE_PAIR_LEN:
|
|
continue
|
|
try:
|
|
numeric = float(value[1])
|
|
except (TypeError, ValueError):
|
|
continue
|
|
output.append({"metric": metric, "value": numeric})
|
|
return output
|
|
|
|
|
|
def _alert_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
for item in entries:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
value = item.get("value")
|
|
name = metric.get("alertname")
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
severity = metric.get("severity") if isinstance(metric.get("severity"), str) else ""
|
|
output.append(
|
|
{
|
|
"alert": name,
|
|
"severity": severity,
|
|
"value": value,
|
|
}
|
|
)
|
|
output.sort(key=lambda item: (-(item.get("value") or 0), item.get("alert") or ""))
|
|
return output
|
|
|
|
|
|
def _vm_alerts_now() -> list[dict[str, Any]]:
|
|
entries = _vm_vector('sum by (alertname,severity) (ALERTS{alertstate="firing"})')
|
|
return _alert_entries(entries)[:_ALERT_TOP_LIMIT]
|
|
|
|
|
|
def _vm_alerts_trend(window: str) -> list[dict[str, Any]]:
|
|
entries = _vm_vector(
|
|
f"topk({_ALERT_TOP_LIMIT}, sum by (alertname,severity) (count_over_time(ALERTS{{alertstate=\"firing\"}}[{window}])))"
|
|
)
|
|
return _alert_entries(entries)
|
|
|
|
|
|
def _alertmanager_alerts(errors: list[str]) -> list[dict[str, Any]]:
|
|
base = settings.alertmanager_url
|
|
if not base:
|
|
return []
|
|
url = f"{base.rstrip('/')}/api/v2/alerts"
|
|
try:
|
|
with httpx.Client(timeout=settings.cluster_state_vm_timeout_sec) as client:
|
|
resp = client.get(url)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
if isinstance(payload, list):
|
|
return [item for item in payload if isinstance(item, dict)]
|
|
except Exception as exc:
|
|
errors.append(f"alertmanager: {exc}")
|
|
return []
|
|
|
|
|
|
def _summarize_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
|
|
items: list[dict[str, Any]] = []
|
|
by_severity: dict[str, int] = {}
|
|
for alert in alerts:
|
|
labels = alert.get("labels") if isinstance(alert.get("labels"), dict) else {}
|
|
alertname = labels.get("alertname")
|
|
if not isinstance(alertname, str) or not alertname:
|
|
continue
|
|
severity = labels.get("severity") if isinstance(labels.get("severity"), str) else ""
|
|
items.append({"alert": alertname, "severity": severity})
|
|
if severity:
|
|
by_severity[severity] = by_severity.get(severity, 0) + 1
|
|
items.sort(key=lambda item: (item.get("severity") or "", item.get("alert") or ""))
|
|
return {
|
|
"total": len(items),
|
|
"by_severity": by_severity,
|
|
"items": items[:_ALERT_TOP_LIMIT],
|
|
}
|
|
|
|
|
|
def _filter_namespace_vector(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
for item in entries:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
namespace = metric.get("namespace")
|
|
if not isinstance(namespace, str) or not namespace:
|
|
continue
|
|
if namespace in _SYSTEM_NAMESPACES:
|
|
continue
|
|
output.append(item)
|
|
return output
|
|
|
|
|
|
def _vm_topk(expr: str, label_key: str) -> dict[str, Any] | None:
|
|
result = _vm_vector(expr)
|
|
if not result:
|
|
return None
|
|
metric = result[0].get("metric") if isinstance(result[0], dict) else {}
|
|
value = result[0].get("value")
|
|
label = metric.get(label_key) if isinstance(metric, dict) else None
|
|
return {"label": label or "", "value": value, "metric": metric}
|
|
|
|
|
|
def _vm_node_metric(expr: str, label_key: str) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
for item in _vm_vector(expr):
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
label = metric.get(label_key)
|
|
value = item.get("value")
|
|
if isinstance(label, str) and label:
|
|
output.append({"node": label, "value": value})
|
|
output.sort(key=lambda item: item.get("node") or "")
|
|
return output
|
|
|
|
|
|
def _vm_baseline_map(expr: str, label_key: str, window: str) -> dict[str, dict[str, float]]:
|
|
averages = _vm_vector(f"avg_over_time(({expr})[{window}])")
|
|
maximums = _vm_vector(f"max_over_time(({expr})[{window}])")
|
|
baseline: dict[str, dict[str, float]] = {}
|
|
for item in averages:
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
label = metric.get(label_key)
|
|
if not isinstance(label, str) or not label:
|
|
continue
|
|
baseline.setdefault(label, {})["avg"] = float(item.get("value") or 0)
|
|
for item in maximums:
|
|
metric = item.get("metric") if isinstance(item.get("metric"), dict) else {}
|
|
label = metric.get(label_key)
|
|
if not isinstance(label, str) or not label:
|
|
continue
|
|
baseline.setdefault(label, {})["max"] = float(item.get("value") or 0)
|
|
return baseline
|
|
|
|
|
|
def _baseline_map_to_list(baseline: dict[str, dict[str, float]], name_key: str) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
for name, stats in baseline.items():
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
output.append(
|
|
{
|
|
name_key: name,
|
|
"avg": stats.get("avg"),
|
|
"max": stats.get("max"),
|
|
}
|
|
)
|
|
output.sort(key=lambda item: (-(item.get("avg") or 0), item.get(name_key) or ""))
|
|
return output
|
|
|
|
|
|
def _limit_entries(entries: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
|
|
if limit <= 0:
|
|
return []
|
|
return entries[:limit]
|
|
|
|
|
|
def _vm_window_series(expr: str, label_key: str, name_key: str, window: str) -> dict[str, list[dict[str, Any]]]:
|
|
avg = _vector_to_named(
|
|
_vm_vector(f"avg_over_time(({expr})[{window}])"),
|
|
label_key,
|
|
name_key,
|
|
)
|
|
max_values = _vector_to_named(
|
|
_vm_vector(f"max_over_time(({expr})[{window}])"),
|
|
label_key,
|
|
name_key,
|
|
)
|
|
p95 = _vector_to_named(
|
|
_vm_vector(f"quantile_over_time(0.95, ({expr})[{window}])"),
|
|
label_key,
|
|
name_key,
|
|
)
|
|
return {"avg": avg, "max": max_values, "p95": p95}
|
|
|
|
|
|
def _trim_window_series(series: dict[str, list[dict[str, Any]]], limit: int) -> dict[str, list[dict[str, Any]]]:
|
|
return {key: _limit_entries(entries, limit) for key, entries in series.items()}
|
|
|
|
|
|
def _build_metric_trends(exprs: dict[str, str], label_key: str, name_key: str, windows: tuple[str, ...], limit: int) -> dict[str, dict[str, dict[str, list[dict[str, Any]]]]]:
|
|
trends: dict[str, dict[str, dict[str, list[dict[str, Any]]]]] = {}
|
|
for metric, expr in exprs.items():
|
|
metric_trends: dict[str, dict[str, list[dict[str, Any]]]] = {}
|
|
for window in windows:
|
|
series = _vm_window_series(expr, label_key, name_key, window)
|
|
metric_trends[window] = _trim_window_series(series, limit)
|
|
trends[metric] = metric_trends
|
|
return trends
|
|
|
|
|
|
def _vm_scalar_window(expr: str, window: str, fn: str) -> float | None:
|
|
return _vm_scalar(f"{fn}(({expr})[{window}])")
|
|
|
|
|
|
def _scalar_trends(expr: str, windows: tuple[str, ...]) -> dict[str, dict[str, float | None]]:
|
|
return {
|
|
window: {
|
|
"avg": _vm_scalar_window(expr, window, "avg_over_time"),
|
|
"min": _vm_scalar_window(expr, window, "min_over_time"),
|
|
"max": _vm_scalar_window(expr, window, "max_over_time"),
|
|
}
|
|
for window in windows
|
|
}
|
|
|
|
|
|
def _cluster_trends() -> dict[str, dict[str, dict[str, float | None]]]:
|
|
exprs = {
|
|
"nodes_ready": 'sum(kube_node_status_condition{condition="Ready",status="true"})',
|
|
"nodes_not_ready": 'sum(kube_node_status_condition{condition="Ready",status="false"})',
|
|
"pods_running": 'sum(kube_pod_status_phase{phase="Running"})',
|
|
"pods_pending": 'sum(kube_pod_status_phase{phase="Pending"})',
|
|
"pods_failed": 'sum(kube_pod_status_phase{phase="Failed"})',
|
|
"pods_succeeded": 'sum(kube_pod_status_phase{phase="Succeeded"})',
|
|
"alerts_firing": 'sum(ALERTS{alertstate="firing"})',
|
|
"cpu_usage": f'sum(rate(container_cpu_usage_seconds_total{{namespace!=""}}[{_RATE_WINDOW}]))',
|
|
"mem_usage": 'sum(container_memory_working_set_bytes{namespace!=""})',
|
|
"net_io": (
|
|
f'sum(rate(container_network_receive_bytes_total{{namespace!=""}}[{_RATE_WINDOW}]) '
|
|
f'+ rate(container_network_transmit_bytes_total{{namespace!=""}}[{_RATE_WINDOW}]))'
|
|
),
|
|
"fs_io": (
|
|
f'sum(rate(container_fs_reads_bytes_total{{namespace!=""}}[{_RATE_WINDOW}]) '
|
|
f'+ rate(container_fs_writes_bytes_total{{namespace!=""}}[{_RATE_WINDOW}]))'
|
|
),
|
|
}
|
|
return {key: _scalar_trends(expr, _TREND_WINDOWS) for key, expr in exprs.items()}
|
|
|
|
|
|
def _node_condition_trends() -> dict[str, dict[str, dict[str, float | None]]]:
|
|
conditions = {
|
|
"ready": 'sum(kube_node_status_condition{condition="Ready",status="true"})',
|
|
"not_ready": 'sum(kube_node_status_condition{condition="Ready",status="false"})',
|
|
"unschedulable": "sum(kube_node_spec_unschedulable)",
|
|
}
|
|
for cond in _PRESSURE_TYPES:
|
|
conditions[cond.lower()] = (
|
|
f'sum(kube_node_status_condition{{condition="{cond}",status="true"}})'
|
|
)
|
|
return {key: _scalar_trends(expr, _TREND_WINDOWS) for key, expr in conditions.items()}
|
|
|
|
__all__ = [name for name in globals() if (name.startswith("_") and not name.startswith("__")) or name in {"ClusterStateSummary", "SignalContext"}]
|