1548 lines
56 KiB
Python
1548 lines
56 KiB
Python
import collections
|
|
import json
|
|
import os
|
|
import re
|
|
import ssl
|
|
import threading
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from typing import Any
|
|
from urllib import error, parse, request
|
|
|
|
BASE = os.environ.get("MATRIX_BASE", "http://othrys-synapse-matrix-synapse:8008")
|
|
AUTH_BASE = os.environ.get("AUTH_BASE", "http://matrix-authentication-service:8080")
|
|
USER = os.environ["BOT_USER"]
|
|
PASSWORD = os.environ["BOT_PASS"]
|
|
ROOM_ALIAS = "#othrys:live.bstein.dev"
|
|
|
|
OLLAMA_URL = os.environ.get("OLLAMA_URL", "https://chat.ai.bstein.dev/")
|
|
MODEL = os.environ.get("OLLAMA_MODEL", "qwen2.5-coder:7b-instruct-q4_0")
|
|
API_KEY = os.environ.get("CHAT_API_KEY", "")
|
|
OLLAMA_TIMEOUT_SEC = float(os.environ.get("OLLAMA_TIMEOUT_SEC", "480"))
|
|
ATLASBOT_HTTP_PORT = int(os.environ.get("ATLASBOT_HTTP_PORT", "8090"))
|
|
ATLASBOT_INTERNAL_TOKEN = os.environ.get("ATLASBOT_INTERNAL_TOKEN") or os.environ.get("CHAT_API_HOMEPAGE", "")
|
|
|
|
KB_DIR = os.environ.get("KB_DIR", "")
|
|
VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428")
|
|
ARIADNE_STATE_URL = os.environ.get("ARIADNE_STATE_URL", "")
|
|
ARIADNE_STATE_TOKEN = os.environ.get("ARIADNE_STATE_TOKEN", "")
|
|
|
|
BOT_MENTIONS = os.environ.get("BOT_MENTIONS", f"{USER},atlas")
|
|
SERVER_NAME = os.environ.get("MATRIX_SERVER_NAME", "live.bstein.dev")
|
|
|
|
MAX_KB_CHARS = int(os.environ.get("ATLASBOT_MAX_KB_CHARS", "2500"))
|
|
MAX_TOOL_CHARS = int(os.environ.get("ATLASBOT_MAX_TOOL_CHARS", "2500"))
|
|
THINKING_INTERVAL_SEC = int(os.environ.get("ATLASBOT_THINKING_INTERVAL_SEC", "120"))
|
|
|
|
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{1,}", re.IGNORECASE)
|
|
HOST_RE = re.compile(r"(?i)([a-z0-9-]+(?:\\.[a-z0-9-]+)+)")
|
|
STOPWORDS = {
|
|
"the",
|
|
"and",
|
|
"for",
|
|
"with",
|
|
"this",
|
|
"that",
|
|
"from",
|
|
"into",
|
|
"what",
|
|
"how",
|
|
"why",
|
|
"when",
|
|
"where",
|
|
"which",
|
|
"who",
|
|
"can",
|
|
"could",
|
|
"should",
|
|
"would",
|
|
"please",
|
|
"help",
|
|
"atlas",
|
|
"othrys",
|
|
}
|
|
|
|
METRIC_HINT_WORDS = {
|
|
"bandwidth",
|
|
"connections",
|
|
"cpu",
|
|
"database",
|
|
"db",
|
|
"disk",
|
|
"health",
|
|
"memory",
|
|
"network",
|
|
"node",
|
|
"nodes",
|
|
"postgres",
|
|
"status",
|
|
"storage",
|
|
"usage",
|
|
"down",
|
|
"slow",
|
|
"error",
|
|
"unknown_error",
|
|
"timeout",
|
|
"crash",
|
|
"crashloop",
|
|
"restart",
|
|
"restarts",
|
|
"pending",
|
|
"unreachable",
|
|
"latency",
|
|
}
|
|
|
|
CODE_FENCE_RE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL)
|
|
TITAN_NODE_RE = re.compile(r"\btitan-[0-9a-z]{2}\b", re.IGNORECASE)
|
|
TITAN_RANGE_RE = re.compile(r"\btitan-([0-9a-z]{2})/([0-9a-z]{2})\b", re.IGNORECASE)
|
|
_DASH_CHARS = "\u2010\u2011\u2012\u2013\u2014\u2015\u2212\uFE63\uFF0D"
|
|
|
|
OPERATION_HINTS = {
|
|
"count": ("how many", "count", "number", "total"),
|
|
"list": ("list", "which", "what are", "show", "names"),
|
|
"top": ("top", "hottest", "highest", "most", "largest", "max", "maximum"),
|
|
"status": ("ready", "not ready", "unready", "down", "missing", "status"),
|
|
}
|
|
|
|
METRIC_HINTS = {
|
|
"cpu": ("cpu",),
|
|
"ram": ("ram", "memory", "mem"),
|
|
"net": ("net", "network", "bandwidth", "throughput"),
|
|
"io": ("io", "disk", "storage"),
|
|
"connections": ("connections", "conn", "postgres", "database", "db"),
|
|
}
|
|
|
|
HARDWARE_HINTS = {
|
|
"amd64": ("amd64", "x86", "x86_64", "x86-64"),
|
|
"jetson": ("jetson",),
|
|
"rpi4": ("rpi4",),
|
|
"rpi5": ("rpi5",),
|
|
"rpi": ("rpi", "raspberry"),
|
|
"arm64": ("arm64", "aarch64"),
|
|
}
|
|
|
|
def normalize_query(text: str) -> str:
|
|
cleaned = (text or "").lower()
|
|
for ch in _DASH_CHARS:
|
|
cleaned = cleaned.replace(ch, "-")
|
|
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
|
return cleaned
|
|
|
|
def _tokens(text: str) -> list[str]:
|
|
toks = [t.lower() for t in TOKEN_RE.findall(text or "")]
|
|
return [t for t in toks if t not in STOPWORDS and len(t) >= 2]
|
|
|
|
|
|
# Mention detection (Matrix rich mentions + plain @atlas).
|
|
MENTION_TOKENS = [m.strip() for m in BOT_MENTIONS.split(",") if m.strip()]
|
|
MENTION_LOCALPARTS = [m.lstrip("@").split(":", 1)[0] for m in MENTION_TOKENS]
|
|
MENTION_RE = re.compile(
|
|
r"(?<!\\w)@(?:" + "|".join(re.escape(m) for m in MENTION_LOCALPARTS) + r")(?:\\:[^\\s]+)?(?!\\w)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
def normalize_user_id(token: str) -> str:
|
|
t = token.strip()
|
|
if not t:
|
|
return ""
|
|
if t.startswith("@") and ":" in t:
|
|
return t
|
|
t = t.lstrip("@")
|
|
if ":" in t:
|
|
return f"@{t}"
|
|
return f"@{t}:{SERVER_NAME}"
|
|
|
|
MENTION_USER_IDS = {normalize_user_id(t).lower() for t in MENTION_TOKENS if normalize_user_id(t)}
|
|
|
|
def _body_mentions_token(body: str) -> bool:
|
|
lower = (body or "").strip().lower()
|
|
if not lower:
|
|
return False
|
|
for token in MENTION_LOCALPARTS:
|
|
for prefix in (token, f"@{token}"):
|
|
if lower.startswith(prefix + ":") or lower.startswith(prefix + ",") or lower.startswith(prefix + " "):
|
|
return True
|
|
return False
|
|
|
|
def is_mentioned(content: dict, body: str) -> bool:
|
|
if MENTION_RE.search(body or "") is not None:
|
|
return True
|
|
if _body_mentions_token(body or ""):
|
|
return True
|
|
mentions = content.get("m.mentions", {})
|
|
user_ids = mentions.get("user_ids", [])
|
|
if not isinstance(user_ids, list):
|
|
return False
|
|
return any(isinstance(uid, str) and uid.lower() in MENTION_USER_IDS for uid in user_ids)
|
|
|
|
|
|
# Matrix HTTP helper.
|
|
def req(method: str, path: str, token: str | None = None, body=None, timeout=60, base: str | None = None):
|
|
url = (base or BASE) + path
|
|
data = None
|
|
headers = {}
|
|
if body is not None:
|
|
data = json.dumps(body).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
r = request.Request(url, data=data, headers=headers, method=method)
|
|
with request.urlopen(r, timeout=timeout) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw.decode()) if raw else {}
|
|
|
|
def login() -> str:
|
|
login_user = normalize_user_id(USER)
|
|
payload = {
|
|
"type": "m.login.password",
|
|
"identifier": {"type": "m.id.user", "user": login_user},
|
|
"password": PASSWORD,
|
|
}
|
|
res = req("POST", "/_matrix/client/v3/login", body=payload, base=AUTH_BASE)
|
|
return res["access_token"]
|
|
|
|
def resolve_alias(token: str, alias: str) -> str:
|
|
enc = parse.quote(alias)
|
|
res = req("GET", f"/_matrix/client/v3/directory/room/{enc}", token)
|
|
return res["room_id"]
|
|
|
|
def join_room(token: str, room: str):
|
|
req("POST", f"/_matrix/client/v3/rooms/{parse.quote(room)}/join", token, body={})
|
|
|
|
def send_msg(token: str, room: str, text: str):
|
|
path = f"/_matrix/client/v3/rooms/{parse.quote(room)}/send/m.room.message"
|
|
req("POST", path, token, body={"msgtype": "m.text", "body": text})
|
|
|
|
|
|
# Atlas KB loader (no external deps; files are pre-rendered JSON via scripts/knowledge_render_atlas.py).
|
|
KB = {"catalog": {}, "runbooks": []}
|
|
_HOST_INDEX: dict[str, list[dict]] = {}
|
|
_NAME_INDEX: set[str] = set()
|
|
_METRIC_INDEX: list[dict[str, Any]] = []
|
|
NODE_REGEX = re.compile(r'node=~"([^"]+)"')
|
|
|
|
def _load_json_file(path: str) -> Any | None:
|
|
try:
|
|
with open(path, "rb") as f:
|
|
return json.loads(f.read().decode("utf-8"))
|
|
except Exception:
|
|
return None
|
|
|
|
def load_kb():
|
|
global KB, _HOST_INDEX, _NAME_INDEX, _METRIC_INDEX
|
|
if not KB_DIR:
|
|
return
|
|
catalog = _load_json_file(os.path.join(KB_DIR, "catalog", "atlas.json")) or {}
|
|
runbooks = _load_json_file(os.path.join(KB_DIR, "catalog", "runbooks.json")) or []
|
|
metrics = _load_json_file(os.path.join(KB_DIR, "catalog", "metrics.json")) or []
|
|
KB = {"catalog": catalog, "runbooks": runbooks}
|
|
|
|
host_index: dict[str, list[dict]] = collections.defaultdict(list)
|
|
for ep in catalog.get("http_endpoints", []) if isinstance(catalog, dict) else []:
|
|
host = (ep.get("host") or "").lower()
|
|
if host:
|
|
host_index[host].append(ep)
|
|
_HOST_INDEX = {k: host_index[k] for k in sorted(host_index.keys())}
|
|
|
|
names: set[str] = set()
|
|
for s in catalog.get("services", []) if isinstance(catalog, dict) else []:
|
|
if isinstance(s, dict) and s.get("name"):
|
|
names.add(str(s["name"]).lower())
|
|
for w in catalog.get("workloads", []) if isinstance(catalog, dict) else []:
|
|
if isinstance(w, dict) and w.get("name"):
|
|
names.add(str(w["name"]).lower())
|
|
_NAME_INDEX = names
|
|
_METRIC_INDEX = metrics if isinstance(metrics, list) else []
|
|
|
|
def _score_kb_docs(query: str) -> list[dict[str, Any]]:
|
|
q = (query or "").strip()
|
|
if not q or not KB.get("runbooks"):
|
|
return []
|
|
ql = q.lower()
|
|
q_tokens = _tokens(q)
|
|
if not q_tokens:
|
|
return []
|
|
|
|
scored: list[tuple[int, dict]] = []
|
|
for doc in KB.get("runbooks", []):
|
|
if not isinstance(doc, dict):
|
|
continue
|
|
title = str(doc.get("title") or "")
|
|
body = str(doc.get("body") or "")
|
|
tags = doc.get("tags") or []
|
|
entrypoints = doc.get("entrypoints") or []
|
|
hay = (title + "\n" + " ".join(tags) + "\n" + " ".join(entrypoints) + "\n" + body).lower()
|
|
score = 0
|
|
for t in set(q_tokens):
|
|
if t in hay:
|
|
score += 3 if t in title.lower() else 1
|
|
for h in entrypoints:
|
|
if isinstance(h, str) and h.lower() in ql:
|
|
score += 4
|
|
if score:
|
|
scored.append((score, doc))
|
|
scored.sort(key=lambda x: x[0], reverse=True)
|
|
return [d for _, d in scored]
|
|
|
|
|
|
def kb_retrieve(query: str, *, limit: int = 3) -> str:
|
|
q = (query or "").strip()
|
|
if not q:
|
|
return ""
|
|
scored = _score_kb_docs(q)
|
|
picked = scored[:limit]
|
|
if not picked:
|
|
return ""
|
|
|
|
parts: list[str] = ["Atlas KB (retrieved):"]
|
|
used = 0
|
|
for d in picked:
|
|
path = d.get("path") or ""
|
|
title = d.get("title") or path
|
|
body = (d.get("body") or "").strip()
|
|
snippet = body[:900].strip()
|
|
chunk = f"- {title} ({path})\n{snippet}"
|
|
if used + len(chunk) > MAX_KB_CHARS:
|
|
break
|
|
parts.append(chunk)
|
|
used += len(chunk)
|
|
return "\n".join(parts).strip()
|
|
|
|
|
|
def kb_retrieve_titles(query: str, *, limit: int = 4) -> str:
|
|
scored = _score_kb_docs(query)
|
|
picked = scored[:limit]
|
|
if not picked:
|
|
return ""
|
|
parts = ["Relevant runbooks:"]
|
|
for doc in picked:
|
|
title = doc.get("title") or doc.get("path") or "runbook"
|
|
path = doc.get("path") or ""
|
|
if path:
|
|
parts.append(f"- {title} ({path})")
|
|
else:
|
|
parts.append(f"- {title}")
|
|
return "\n".join(parts)
|
|
|
|
def _extract_titan_nodes(text: str) -> list[str]:
|
|
cleaned = normalize_query(text)
|
|
names = {n.lower() for n in TITAN_NODE_RE.findall(cleaned) if n}
|
|
for match in re.finditer(r"titan-([0-9a-z]{2}(?:[/,][0-9a-z]{2})+)", cleaned, re.IGNORECASE):
|
|
tail = match.group(1)
|
|
for part in re.split(r"[/,]", tail):
|
|
part = part.strip()
|
|
if part:
|
|
names.add(f"titan-{part.lower()}")
|
|
for match in TITAN_RANGE_RE.finditer(cleaned):
|
|
left, right = match.groups()
|
|
if left:
|
|
names.add(f"titan-{left.lower()}")
|
|
if right:
|
|
names.add(f"titan-{right.lower()}")
|
|
return sorted(names)
|
|
|
|
def _humanize_rate(value: str, *, unit: str) -> str:
|
|
try:
|
|
val = float(value)
|
|
except (TypeError, ValueError):
|
|
return value
|
|
if unit == "%":
|
|
return f"{val:.1f}%"
|
|
if val >= 1024 * 1024:
|
|
return f"{val / (1024 * 1024):.2f} MB/s"
|
|
if val >= 1024:
|
|
return f"{val / 1024:.2f} KB/s"
|
|
return f"{val:.2f} B/s"
|
|
|
|
def _has_any(text: str, phrases: tuple[str, ...]) -> bool:
|
|
return any(p in text for p in phrases)
|
|
|
|
def _detect_operation(q: str) -> str | None:
|
|
if _has_any(q, OPERATION_HINTS["top"]):
|
|
return "top"
|
|
for op, phrases in OPERATION_HINTS.items():
|
|
if op == "top":
|
|
continue
|
|
if _has_any(q, phrases):
|
|
return op
|
|
return None
|
|
|
|
def _detect_metric(q: str) -> str | None:
|
|
for metric, phrases in METRIC_HINTS.items():
|
|
if _has_any(q, phrases):
|
|
return metric
|
|
return None
|
|
|
|
def _detect_hardware_filters(q: str) -> tuple[set[str], set[str]]:
|
|
include: set[str] = set()
|
|
exclude: set[str] = set()
|
|
for hardware, phrases in HARDWARE_HINTS.items():
|
|
for phrase in phrases:
|
|
if f"non {phrase}" in q or f"non-{phrase}" in q or f"not {phrase}" in q:
|
|
exclude.add(hardware)
|
|
elif phrase in q:
|
|
include.add(hardware)
|
|
return include, exclude
|
|
|
|
def _detect_entity(q: str) -> str | None:
|
|
if "node" in q or "nodes" in q or "worker" in q or TITAN_NODE_RE.search(q):
|
|
return "node"
|
|
if "pod" in q or "pods" in q:
|
|
return "pod"
|
|
if "namespace" in q or "namespaces" in q:
|
|
return "namespace"
|
|
return None
|
|
|
|
def _metric_entry_score(entry: dict[str, Any], tokens: list[str], *, metric: str | None, op: str | None) -> int:
|
|
hay = _metric_tokens(entry)
|
|
score = 0
|
|
for t in set(tokens):
|
|
if t in hay:
|
|
score += 2 if t in (entry.get("panel_title") or "").lower() else 1
|
|
if metric:
|
|
for phrase in METRIC_HINTS.get(metric, (metric,)):
|
|
if phrase in hay:
|
|
score += 3
|
|
if op == "top" and ("hottest" in hay or "top" in hay):
|
|
score += 3
|
|
if "node" in hay:
|
|
score += 1
|
|
return score
|
|
|
|
def _select_metric_entry(tokens: list[str], *, metric: str | None, op: str | None) -> dict[str, Any] | None:
|
|
scored: list[tuple[int, dict[str, Any]]] = []
|
|
for entry in _METRIC_INDEX:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
score = _metric_entry_score(entry, tokens, metric=metric, op=op)
|
|
if score:
|
|
scored.append((score, entry))
|
|
if not scored:
|
|
return None
|
|
scored.sort(key=lambda item: item[0], reverse=True)
|
|
return scored[0][1]
|
|
|
|
def _apply_node_filter(expr: str, node_regex: str | None) -> str:
|
|
if not node_regex:
|
|
return expr
|
|
needle = 'node_uname_info{nodename!=""}'
|
|
replacement = f'node_uname_info{{nodename!=\"\",nodename=~\"{node_regex}\"}}'
|
|
return expr.replace(needle, replacement)
|
|
|
|
def _metric_expr_uses_percent(entry: dict[str, Any]) -> bool:
|
|
exprs = entry.get("exprs")
|
|
expr = exprs[0] if isinstance(exprs, list) and exprs else ""
|
|
return "* 100" in expr or "*100" in expr
|
|
|
|
|
|
def _format_metric_value(value: str, *, percent: bool) -> str:
|
|
try:
|
|
num = float(value)
|
|
except (TypeError, ValueError):
|
|
return value
|
|
if percent:
|
|
return f"{num:.1f}%"
|
|
if abs(num) >= 1:
|
|
return f"{num:.2f}".rstrip("0").rstrip(".")
|
|
return f"{num:.4f}".rstrip("0").rstrip(".")
|
|
|
|
|
|
def _format_metric_label(metric: dict[str, Any]) -> str:
|
|
label_parts = []
|
|
for k in ("namespace", "pod", "container", "node", "instance", "job", "phase"):
|
|
if metric.get(k):
|
|
label_parts.append(f"{k}={metric.get(k)}")
|
|
if not label_parts:
|
|
for k in sorted(metric.keys()):
|
|
if k.startswith("__"):
|
|
continue
|
|
label_parts.append(f"{k}={metric.get(k)}")
|
|
if len(label_parts) >= 4:
|
|
break
|
|
return ", ".join(label_parts) if label_parts else "series"
|
|
|
|
|
|
def _primary_series_metric(res: dict | None) -> tuple[str | None, str | None]:
|
|
series = _vm_value_series(res or {})
|
|
if not series:
|
|
return (None, None)
|
|
first = series[0]
|
|
metric = first.get("metric") if isinstance(first, dict) else {}
|
|
value = first.get("value") if isinstance(first, dict) else []
|
|
node = metric.get("node") if isinstance(metric, dict) else None
|
|
val = value[1] if isinstance(value, list) and len(value) > 1 else None
|
|
return (node, val)
|
|
|
|
|
|
def _format_metric_answer(entry: dict[str, Any], res: dict | None) -> str:
|
|
series = _vm_value_series(res)
|
|
panel = entry.get("panel_title") or "Metric"
|
|
if not series:
|
|
return ""
|
|
percent = _metric_expr_uses_percent(entry)
|
|
lines: list[str] = []
|
|
for r in series[:5]:
|
|
if not isinstance(r, dict):
|
|
continue
|
|
metric = r.get("metric") or {}
|
|
value = r.get("value") or []
|
|
val = value[1] if isinstance(value, list) and len(value) > 1 else ""
|
|
label = _format_metric_label(metric if isinstance(metric, dict) else {})
|
|
lines.append(f"{label}: {_format_metric_value(val, percent=percent)}")
|
|
if not lines:
|
|
return ""
|
|
if len(lines) == 1:
|
|
return f"{panel}: {lines[0]}."
|
|
return f"{panel}:\n" + "\n".join(f"- {line}" for line in lines)
|
|
|
|
def _inventory_filter(
|
|
inventory: list[dict[str, Any]],
|
|
*,
|
|
include_hw: set[str],
|
|
exclude_hw: set[str],
|
|
only_workers: bool,
|
|
only_ready: bool | None,
|
|
nodes_in_query: list[str],
|
|
) -> list[dict[str, Any]]:
|
|
results = inventory
|
|
if nodes_in_query:
|
|
results = [node for node in results if node.get("name") in nodes_in_query]
|
|
if only_workers:
|
|
results = [node for node in results if node.get("is_worker") is True]
|
|
if only_ready is True:
|
|
results = [node for node in results if node.get("ready") is True]
|
|
if only_ready is False:
|
|
results = [node for node in results if node.get("ready") is False]
|
|
if include_hw:
|
|
results = [node for node in results if _hardware_match(node, include_hw)]
|
|
if exclude_hw:
|
|
results = [node for node in results if not _hardware_match(node, exclude_hw)]
|
|
return results
|
|
|
|
def _hardware_match(node: dict[str, Any], filters: set[str]) -> bool:
|
|
hw = node.get("hardware") or ""
|
|
arch = node.get("arch") or ""
|
|
for f in filters:
|
|
if f == "rpi" and hw in ("rpi4", "rpi5"):
|
|
return True
|
|
if f == "arm64" and arch == "arm64":
|
|
return True
|
|
if hw == f:
|
|
return True
|
|
if f == "amd64" and arch == "amd64":
|
|
return True
|
|
return False
|
|
|
|
def _node_roles(labels: dict[str, Any]) -> list[str]:
|
|
roles: list[str] = []
|
|
for key in labels.keys():
|
|
if key.startswith("node-role.kubernetes.io/"):
|
|
role = key.split("/", 1)[-1]
|
|
if role:
|
|
roles.append(role)
|
|
return sorted(set(roles))
|
|
|
|
def _hardware_class(labels: dict[str, Any]) -> str:
|
|
if str(labels.get("jetson") or "").lower() == "true":
|
|
return "jetson"
|
|
hardware = (labels.get("hardware") or "").strip().lower()
|
|
if hardware in ("rpi4", "rpi5"):
|
|
return hardware
|
|
arch = labels.get("kubernetes.io/arch") or labels.get("beta.kubernetes.io/arch") or ""
|
|
if arch == "amd64":
|
|
return "amd64"
|
|
if arch == "arm64":
|
|
return "arm64-unknown"
|
|
return "unknown"
|
|
|
|
def node_inventory_live() -> list[dict[str, Any]]:
|
|
try:
|
|
data = k8s_get("/api/v1/nodes?limit=500")
|
|
except Exception:
|
|
return []
|
|
items = data.get("items") or []
|
|
inventory: list[dict[str, Any]] = []
|
|
for node in items if isinstance(items, list) else []:
|
|
meta = node.get("metadata") or {}
|
|
labels = meta.get("labels") or {}
|
|
name = meta.get("name") or ""
|
|
if not name:
|
|
continue
|
|
inventory.append(
|
|
{
|
|
"name": name,
|
|
"arch": labels.get("kubernetes.io/arch") or labels.get("beta.kubernetes.io/arch") or "",
|
|
"hardware": _hardware_class(labels),
|
|
"roles": _node_roles(labels),
|
|
"is_worker": _node_is_worker(node),
|
|
"ready": _node_ready_status(node),
|
|
}
|
|
)
|
|
return sorted(inventory, key=lambda item: item["name"])
|
|
|
|
def _group_nodes(inventory: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
grouped: dict[str, list[str]] = collections.defaultdict(list)
|
|
for node in inventory:
|
|
grouped[node.get("hardware") or "unknown"].append(node["name"])
|
|
return {k: sorted(v) for k, v in grouped.items()}
|
|
|
|
def node_inventory_context(query: str, inventory: list[dict[str, Any]] | None = None) -> str:
|
|
q = normalize_query(query)
|
|
if not any(word in q for word in ("node", "nodes", "raspberry", "rpi", "jetson", "amd64", "hardware", "cluster")):
|
|
return ""
|
|
if inventory is None:
|
|
inventory = node_inventory_live()
|
|
if not inventory:
|
|
return ""
|
|
groups = _group_nodes(inventory)
|
|
total = len(inventory)
|
|
ready = sum(1 for node in inventory if node.get("ready") is True)
|
|
not_ready = sum(1 for node in inventory if node.get("ready") is False)
|
|
lines: list[str] = [
|
|
"Node inventory (live):",
|
|
f"- total: {total}, ready: {ready}, not ready: {not_ready}",
|
|
]
|
|
for key in ("rpi5", "rpi4", "jetson", "amd64", "arm64-unknown", "unknown"):
|
|
if key in groups:
|
|
lines.append(f"- {key}: {', '.join(groups[key])}")
|
|
non_rpi = sorted(set(groups.get("jetson", [])) | set(groups.get("amd64", [])))
|
|
if non_rpi:
|
|
lines.append(f"- non_raspberry_pi (derived): {', '.join(non_rpi)}")
|
|
unknowns = groups.get("arm64-unknown", []) + groups.get("unknown", [])
|
|
if unknowns:
|
|
lines.append("- note: nodes labeled arm64-unknown/unknown may still be Raspberry Pi unless tagged.")
|
|
expected_workers = expected_worker_nodes_from_metrics()
|
|
if expected_workers:
|
|
ready_workers, not_ready_workers = worker_nodes_status()
|
|
missing = sorted(set(expected_workers) - set(ready_workers + not_ready_workers))
|
|
lines.append(f"- expected_workers (grafana): {', '.join(expected_workers)}")
|
|
lines.append(f"- workers_ready: {', '.join(ready_workers)}")
|
|
if not_ready_workers:
|
|
lines.append(f"- workers_not_ready: {', '.join(not_ready_workers)}")
|
|
if missing:
|
|
lines.append(f"- workers_missing (derived): {', '.join(missing)}")
|
|
return "\n".join(lines)
|
|
|
|
def node_inventory_for_prompt(prompt: str) -> list[dict[str, Any]]:
|
|
q = normalize_query(prompt)
|
|
if any(word in q for word in ("node", "nodes", "raspberry", "rpi", "jetson", "amd64", "hardware", "cluster", "worker")):
|
|
return node_inventory_live()
|
|
return []
|
|
|
|
def _inventory_sets(inventory: list[dict[str, Any]]) -> dict[str, Any]:
|
|
names = [node["name"] for node in inventory]
|
|
ready = [node["name"] for node in inventory if node.get("ready") is True]
|
|
not_ready = [node["name"] for node in inventory if node.get("ready") is False]
|
|
groups = _group_nodes(inventory)
|
|
workers = [node for node in inventory if node.get("is_worker") is True]
|
|
worker_names = [node["name"] for node in workers]
|
|
worker_ready = [node["name"] for node in workers if node.get("ready") is True]
|
|
worker_not_ready = [node["name"] for node in workers if node.get("ready") is False]
|
|
expected_workers = expected_worker_nodes_from_metrics()
|
|
expected_ready = [n for n in expected_workers if n in ready] if expected_workers else []
|
|
expected_not_ready = [n for n in expected_workers if n in not_ready] if expected_workers else []
|
|
expected_missing = [n for n in expected_workers if n not in names] if expected_workers else []
|
|
return {
|
|
"names": sorted(names),
|
|
"ready": sorted(ready),
|
|
"not_ready": sorted(not_ready),
|
|
"groups": groups,
|
|
"worker_names": sorted(worker_names),
|
|
"worker_ready": sorted(worker_ready),
|
|
"worker_not_ready": sorted(worker_not_ready),
|
|
"expected_workers": expected_workers,
|
|
"expected_ready": sorted(expected_ready),
|
|
"expected_not_ready": sorted(expected_not_ready),
|
|
"expected_missing": sorted(expected_missing),
|
|
}
|
|
|
|
def structured_answer(prompt: str, *, inventory: list[dict[str, Any]], metrics_summary: str) -> str:
|
|
q = normalize_query(prompt)
|
|
if not q:
|
|
return ""
|
|
|
|
tokens = _tokens(q)
|
|
op = _detect_operation(q)
|
|
metric = _detect_metric(q)
|
|
entity = _detect_entity(q)
|
|
include_hw, exclude_hw = _detect_hardware_filters(q)
|
|
nodes_in_query = _extract_titan_nodes(q)
|
|
only_workers = "worker" in q or "workers" in q
|
|
only_ready: bool | None = None
|
|
if "not ready" in q or "unready" in q or "down" in q or "missing" in q:
|
|
only_ready = False
|
|
elif "ready" in q:
|
|
only_ready = True
|
|
|
|
if entity == "node" and only_ready is not None and op != "count":
|
|
op = "status"
|
|
|
|
if not op and entity == "node":
|
|
op = "list" if (include_hw or exclude_hw or nodes_in_query) else "count"
|
|
|
|
if op == "top" and metric is None:
|
|
metric = "cpu"
|
|
|
|
# Metrics-first when a metric or top operation is requested.
|
|
if metric or op == "top":
|
|
entry = _select_metric_entry(tokens, metric=metric, op=op)
|
|
if entry and isinstance(entry.get("exprs"), list) and entry["exprs"]:
|
|
expr = entry["exprs"][0]
|
|
if inventory:
|
|
scoped = _inventory_filter(
|
|
inventory,
|
|
include_hw=include_hw,
|
|
exclude_hw=exclude_hw,
|
|
only_workers=only_workers,
|
|
only_ready=None,
|
|
nodes_in_query=nodes_in_query,
|
|
)
|
|
if scoped:
|
|
node_regex = "|".join([n["name"] for n in scoped])
|
|
expr = _apply_node_filter(expr, node_regex)
|
|
res = vm_query(expr, timeout=20)
|
|
answer = _format_metric_answer(entry, res)
|
|
if answer:
|
|
scope_parts: list[str] = []
|
|
if include_hw:
|
|
scope_parts.append(" and ".join(sorted(include_hw)))
|
|
if exclude_hw:
|
|
scope_parts.append(f"excluding {' and '.join(sorted(exclude_hw))}")
|
|
if only_workers:
|
|
scope_parts.append("worker")
|
|
if scope_parts:
|
|
scope = " ".join(scope_parts)
|
|
overall_note = ""
|
|
base_res = vm_query(entry["exprs"][0], timeout=20)
|
|
base_node, base_val = _primary_series_metric(base_res)
|
|
scoped_node, scoped_val = _primary_series_metric(res)
|
|
if base_node and scoped_node and base_node != scoped_node:
|
|
percent = _metric_expr_uses_percent(entry)
|
|
base_val_fmt = _format_metric_value(base_val or "", percent=percent)
|
|
overall_note = f" Overall hottest node: {base_node} ({base_val_fmt})."
|
|
return f"Among {scope} nodes, {answer}{overall_note}"
|
|
return answer
|
|
if metrics_summary:
|
|
return metrics_summary
|
|
|
|
if entity != "node" or not inventory:
|
|
if any(word in q for word in METRIC_HINT_WORDS) and not metrics_summary:
|
|
return "I don't have data to answer that right now."
|
|
return ""
|
|
|
|
expected_workers = expected_worker_nodes_from_metrics()
|
|
filtered = _inventory_filter(
|
|
inventory,
|
|
include_hw=include_hw,
|
|
exclude_hw=exclude_hw,
|
|
only_workers=only_workers,
|
|
only_ready=only_ready if op in ("status", "count") else None,
|
|
nodes_in_query=nodes_in_query,
|
|
)
|
|
names = [node["name"] for node in filtered]
|
|
|
|
if op == "status":
|
|
if "missing" in q and expected_workers:
|
|
missing = sorted(set(expected_workers) - {n["name"] for n in inventory})
|
|
return "Missing nodes: " + (", ".join(missing) if missing else "none") + "."
|
|
if only_ready is False:
|
|
return "Not ready nodes: " + (", ".join(names) if names else "none") + "."
|
|
if only_ready is True:
|
|
return f"Ready nodes ({len(names)}): " + (", ".join(names) if names else "none") + "."
|
|
|
|
if op == "count":
|
|
if expected_workers and ("expected" in q or "should" in q):
|
|
missing = sorted(set(expected_workers) - {n["name"] for n in inventory})
|
|
msg = f"Grafana inventory expects {len(expected_workers)} worker nodes."
|
|
if missing:
|
|
msg += f" Missing: {', '.join(missing)}."
|
|
return msg
|
|
if not (include_hw or exclude_hw or nodes_in_query or only_workers):
|
|
return f"Atlas has {len(names)} nodes."
|
|
return f"Matching nodes: {len(names)}."
|
|
|
|
if op == "list":
|
|
if nodes_in_query:
|
|
parts = []
|
|
existing = {n["name"] for n in inventory}
|
|
for node in nodes_in_query:
|
|
parts.append(f"{node}: {'present' if node in existing else 'not present'}")
|
|
return "Node presence: " + ", ".join(parts) + "."
|
|
if not names:
|
|
return "Matching nodes: none."
|
|
shown = names[:30]
|
|
suffix = f", … (+{len(names) - 30} more)" if len(names) > 30 else ""
|
|
return "Matching nodes: " + ", ".join(shown) + suffix + "."
|
|
|
|
return ""
|
|
|
|
def _metric_tokens(entry: dict[str, Any]) -> str:
|
|
parts: list[str] = []
|
|
for key in ("panel_title", "dashboard", "description"):
|
|
val = entry.get(key)
|
|
if isinstance(val, str) and val:
|
|
parts.append(val.lower())
|
|
tags = entry.get("tags")
|
|
if isinstance(tags, list):
|
|
parts.extend(str(t).lower() for t in tags if t)
|
|
return " ".join(parts)
|
|
|
|
def metrics_lookup(query: str, limit: int = 3) -> list[dict[str, Any]]:
|
|
q_tokens = _tokens(query)
|
|
if not q_tokens or not _METRIC_INDEX:
|
|
return []
|
|
scored: list[tuple[int, dict[str, Any]]] = []
|
|
for entry in _METRIC_INDEX:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
hay = _metric_tokens(entry)
|
|
if not hay:
|
|
continue
|
|
score = 0
|
|
for t in set(q_tokens):
|
|
if t in hay:
|
|
score += 2 if t in (entry.get("panel_title") or "").lower() else 1
|
|
if score:
|
|
scored.append((score, entry))
|
|
scored.sort(key=lambda item: item[0], reverse=True)
|
|
return [entry for _, entry in scored[:limit]]
|
|
|
|
def metrics_query_context(prompt: str, *, allow_tools: bool) -> tuple[str, str]:
|
|
if not allow_tools:
|
|
return "", ""
|
|
lower = (prompt or "").lower()
|
|
if not any(word in lower for word in METRIC_HINT_WORDS):
|
|
return "", ""
|
|
matches = metrics_lookup(prompt, limit=1)
|
|
if not matches:
|
|
return "", ""
|
|
entry = matches[0]
|
|
dashboard = entry.get("dashboard") or "dashboard"
|
|
panel = entry.get("panel_title") or "panel"
|
|
exprs = entry.get("exprs") if isinstance(entry.get("exprs"), list) else []
|
|
if not exprs:
|
|
return "", ""
|
|
rendered_parts: list[str] = []
|
|
for expr in exprs[:2]:
|
|
res = vm_query(expr, timeout=20)
|
|
rendered = vm_render_result(res, limit=10)
|
|
if rendered:
|
|
rendered_parts.append(rendered)
|
|
if not rendered_parts:
|
|
return "", f"{panel}: matched dashboard panel but VictoriaMetrics did not return data."
|
|
summary = "\n".join(rendered_parts)
|
|
context = f"Metrics (from {dashboard} / {panel}):\n{summary}"
|
|
fallback = _metrics_fallback_summary(panel, summary)
|
|
return context, fallback
|
|
|
|
def catalog_hints(query: str) -> tuple[str, list[tuple[str, str]]]:
|
|
q = (query or "").strip()
|
|
if not q or not KB.get("catalog"):
|
|
return "", []
|
|
ql = q.lower()
|
|
hosts = {m.group(1).lower() for m in HOST_RE.finditer(ql) if m.group(1).lower().endswith("bstein.dev")}
|
|
|
|
# Also match by known workload/service names.
|
|
for t in _tokens(ql):
|
|
if t in _NAME_INDEX:
|
|
hosts |= {ep["host"].lower() for ep in KB["catalog"].get("http_endpoints", []) if isinstance(ep, dict) and ep.get("backend", {}).get("service") == t}
|
|
|
|
edges: list[tuple[str, str]] = []
|
|
lines: list[str] = []
|
|
for host in sorted(hosts):
|
|
for ep in _HOST_INDEX.get(host, []):
|
|
backend = ep.get("backend") or {}
|
|
ns = backend.get("namespace") or ""
|
|
svc = backend.get("service") or ""
|
|
path = ep.get("path") or "/"
|
|
if not svc:
|
|
continue
|
|
wk = backend.get("workloads") or []
|
|
wk_str = ", ".join(f"{w.get('kind')}:{w.get('name')}" for w in wk if isinstance(w, dict) and w.get("name")) or "unknown"
|
|
lines.append(f"- {host}{path} → {ns}/{svc} → {wk_str}")
|
|
for w in wk:
|
|
if isinstance(w, dict) and w.get("name"):
|
|
edges.append((ns, str(w["name"])))
|
|
if not lines:
|
|
return "", []
|
|
return "Atlas endpoints (from GitOps):\n" + "\n".join(lines[:20]), edges
|
|
|
|
|
|
# Kubernetes API (read-only). RBAC is provided via ServiceAccount atlasbot.
|
|
_K8S_TOKEN: str | None = None
|
|
_K8S_CTX: ssl.SSLContext | None = None
|
|
|
|
def _k8s_context() -> ssl.SSLContext:
|
|
global _K8S_CTX
|
|
if _K8S_CTX is not None:
|
|
return _K8S_CTX
|
|
ca_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
ctx = ssl.create_default_context(cafile=ca_path)
|
|
_K8S_CTX = ctx
|
|
return ctx
|
|
|
|
def _k8s_token() -> str:
|
|
global _K8S_TOKEN
|
|
if _K8S_TOKEN:
|
|
return _K8S_TOKEN
|
|
token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
with open(token_path, "r", encoding="utf-8") as f:
|
|
_K8S_TOKEN = f.read().strip()
|
|
return _K8S_TOKEN
|
|
|
|
def k8s_get(path: str, timeout: int = 8) -> dict:
|
|
host = os.environ.get("KUBERNETES_SERVICE_HOST")
|
|
port = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS") or os.environ.get("KUBERNETES_SERVICE_PORT") or "443"
|
|
if not host:
|
|
raise RuntimeError("k8s host missing")
|
|
url = f"https://{host}:{port}{path}"
|
|
headers = {"Authorization": f"Bearer {_k8s_token()}"}
|
|
r = request.Request(url, headers=headers, method="GET")
|
|
with request.urlopen(r, timeout=timeout, context=_k8s_context()) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw.decode()) if raw else {}
|
|
|
|
def _ariadne_state(timeout: int = 5) -> dict | None:
|
|
if not ARIADNE_STATE_URL:
|
|
return None
|
|
headers = {}
|
|
if ARIADNE_STATE_TOKEN:
|
|
headers["X-Internal-Token"] = ARIADNE_STATE_TOKEN
|
|
r = request.Request(ARIADNE_STATE_URL, headers=headers, method="GET")
|
|
try:
|
|
with request.urlopen(r, timeout=timeout) as resp:
|
|
raw = resp.read()
|
|
payload = json.loads(raw.decode()) if raw else {}
|
|
return payload if isinstance(payload, dict) else None
|
|
except Exception:
|
|
return None
|
|
|
|
def k8s_pods(namespace: str) -> list[dict]:
|
|
data = k8s_get(f"/api/v1/namespaces/{parse.quote(namespace)}/pods?limit=500")
|
|
items = data.get("items") or []
|
|
return items if isinstance(items, list) else []
|
|
|
|
def summarize_pods(namespace: str, prefixes: set[str] | None = None) -> str:
|
|
try:
|
|
pods = k8s_pods(namespace)
|
|
except Exception:
|
|
return ""
|
|
out: list[str] = []
|
|
for p in pods:
|
|
md = p.get("metadata") or {}
|
|
st = p.get("status") or {}
|
|
name = md.get("name") or ""
|
|
if prefixes and not any(name.startswith(pref + "-") or name == pref or name.startswith(pref) for pref in prefixes):
|
|
continue
|
|
phase = st.get("phase") or "?"
|
|
cs = st.get("containerStatuses") or []
|
|
restarts = 0
|
|
ready = 0
|
|
total = 0
|
|
reason = st.get("reason") or ""
|
|
for c in cs if isinstance(cs, list) else []:
|
|
if not isinstance(c, dict):
|
|
continue
|
|
total += 1
|
|
restarts += int(c.get("restartCount") or 0)
|
|
if c.get("ready"):
|
|
ready += 1
|
|
state = c.get("state") or {}
|
|
if not reason and isinstance(state, dict):
|
|
waiting = state.get("waiting") or {}
|
|
if isinstance(waiting, dict) and waiting.get("reason"):
|
|
reason = waiting.get("reason")
|
|
extra = f" ({reason})" if reason else ""
|
|
out.append(f"- {namespace}/{name}: {phase} {ready}/{total} restarts={restarts}{extra}")
|
|
return "\n".join(out[:20])
|
|
|
|
def flux_not_ready() -> str:
|
|
try:
|
|
data = k8s_get(
|
|
"/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/flux-system/kustomizations?limit=200"
|
|
)
|
|
except Exception:
|
|
return ""
|
|
items = data.get("items") or []
|
|
bad: list[str] = []
|
|
for it in items if isinstance(items, list) else []:
|
|
md = it.get("metadata") or {}
|
|
st = it.get("status") or {}
|
|
name = md.get("name") or ""
|
|
conds = st.get("conditions") or []
|
|
ready = None
|
|
msg = ""
|
|
for c in conds if isinstance(conds, list) else []:
|
|
if isinstance(c, dict) and c.get("type") == "Ready":
|
|
ready = c.get("status")
|
|
msg = c.get("message") or ""
|
|
if ready not in ("True", True):
|
|
bad.append(f"- flux kustomization/{name}: Ready={ready} {msg}".strip())
|
|
return "\n".join(bad[:10])
|
|
|
|
|
|
# VictoriaMetrics (PromQL) helpers.
|
|
def vm_query(query: str, timeout: int = 8) -> dict | None:
|
|
try:
|
|
url = VM_URL.rstrip("/") + "/api/v1/query?" + parse.urlencode({"query": query})
|
|
with request.urlopen(url, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode())
|
|
except Exception:
|
|
return None
|
|
|
|
def _vm_value_series(res: dict) -> list[dict]:
|
|
if not res or (res.get("status") != "success"):
|
|
return []
|
|
data = res.get("data") or {}
|
|
result = data.get("result") or []
|
|
return result if isinstance(result, list) else []
|
|
|
|
def vm_render_result(res: dict | None, limit: int = 12) -> str:
|
|
if not res:
|
|
return ""
|
|
series = _vm_value_series(res)
|
|
if not series:
|
|
return ""
|
|
out: list[str] = []
|
|
for r in series[:limit]:
|
|
if not isinstance(r, dict):
|
|
continue
|
|
metric = r.get("metric") or {}
|
|
value = r.get("value") or []
|
|
val = value[1] if isinstance(value, list) and len(value) > 1 else ""
|
|
# Prefer common labels if present.
|
|
label_parts = []
|
|
for k in ("namespace", "pod", "container", "node", "instance", "job", "phase"):
|
|
if isinstance(metric, dict) and metric.get(k):
|
|
label_parts.append(f"{k}={metric.get(k)}")
|
|
if not label_parts and isinstance(metric, dict):
|
|
for k in sorted(metric.keys()):
|
|
if k.startswith("__"):
|
|
continue
|
|
label_parts.append(f"{k}={metric.get(k)}")
|
|
if len(label_parts) >= 4:
|
|
break
|
|
labels = ", ".join(label_parts) if label_parts else "series"
|
|
out.append(f"- {labels}: {val}")
|
|
return "\n".join(out)
|
|
|
|
def _parse_metric_lines(summary: str) -> dict[str, str]:
|
|
parsed: dict[str, str] = {}
|
|
for line in (summary or "").splitlines():
|
|
line = line.strip()
|
|
if not line.startswith("-"):
|
|
continue
|
|
try:
|
|
label, value = line.lstrip("-").split(":", 1)
|
|
except ValueError:
|
|
continue
|
|
parsed[label.strip()] = value.strip()
|
|
return parsed
|
|
|
|
def _metrics_fallback_summary(panel: str, summary: str) -> str:
|
|
parsed = _parse_metric_lines(summary)
|
|
panel_l = (panel or "").lower()
|
|
if parsed:
|
|
items = list(parsed.items())
|
|
if len(items) == 1:
|
|
label, value = items[0]
|
|
return f"{panel}: {label} = {value}."
|
|
compact = "; ".join(f"{k}={v}" for k, v in items)
|
|
return f"{panel}: {compact}."
|
|
if panel_l:
|
|
return f"{panel}: {summary}"
|
|
return summary
|
|
|
|
def _node_ready_status(node: dict) -> bool | None:
|
|
conditions = node.get("status", {}).get("conditions") or []
|
|
for cond in conditions if isinstance(conditions, list) else []:
|
|
if cond.get("type") == "Ready":
|
|
if cond.get("status") == "True":
|
|
return True
|
|
if cond.get("status") == "False":
|
|
return False
|
|
return None
|
|
return None
|
|
|
|
def _node_is_worker(node: dict) -> bool:
|
|
labels = (node.get("metadata") or {}).get("labels") or {}
|
|
if labels.get("node-role.kubernetes.io/control-plane") is not None:
|
|
return False
|
|
if labels.get("node-role.kubernetes.io/master") is not None:
|
|
return False
|
|
if labels.get("node-role.kubernetes.io/worker") is not None:
|
|
return True
|
|
return True
|
|
|
|
def worker_nodes_status() -> tuple[list[str], list[str]]:
|
|
try:
|
|
data = k8s_get("/api/v1/nodes?limit=500")
|
|
except Exception:
|
|
return ([], [])
|
|
items = data.get("items") or []
|
|
ready_nodes: list[str] = []
|
|
not_ready_nodes: list[str] = []
|
|
for node in items if isinstance(items, list) else []:
|
|
if not _node_is_worker(node):
|
|
continue
|
|
name = (node.get("metadata") or {}).get("name") or ""
|
|
if not name:
|
|
continue
|
|
ready = _node_ready_status(node)
|
|
if ready is True:
|
|
ready_nodes.append(name)
|
|
elif ready is False:
|
|
not_ready_nodes.append(name)
|
|
return (sorted(ready_nodes), sorted(not_ready_nodes))
|
|
|
|
def expected_worker_nodes_from_metrics() -> list[str]:
|
|
for entry in _METRIC_INDEX:
|
|
panel = (entry.get("panel_title") or "").lower()
|
|
if "worker nodes ready" not in panel:
|
|
continue
|
|
exprs = entry.get("exprs") if isinstance(entry.get("exprs"), list) else []
|
|
for expr in exprs:
|
|
if not isinstance(expr, str):
|
|
continue
|
|
match = NODE_REGEX.search(expr)
|
|
if not match:
|
|
continue
|
|
raw = match.group(1)
|
|
nodes = [n.strip() for n in raw.split("|") if n.strip()]
|
|
return sorted(nodes)
|
|
return []
|
|
|
|
def _context_fallback(context: str) -> str:
|
|
if not context:
|
|
return ""
|
|
trimmed = context.strip()
|
|
if len(trimmed) > MAX_TOOL_CHARS:
|
|
trimmed = trimmed[: MAX_TOOL_CHARS - 3].rstrip() + "..."
|
|
return "Here is what I found:\n" + trimmed
|
|
|
|
def vm_top_restarts(hours: int = 1) -> str:
|
|
q = f"topk(5, sum by (namespace,pod) (increase(kube_pod_container_status_restarts_total[{hours}h])))"
|
|
res = vm_query(q)
|
|
if not res or (res.get("status") != "success"):
|
|
return ""
|
|
out: list[str] = []
|
|
for r in (res.get("data") or {}).get("result") or []:
|
|
if not isinstance(r, dict):
|
|
continue
|
|
m = r.get("metric") or {}
|
|
v = r.get("value") or []
|
|
ns = (m.get("namespace") or "").strip()
|
|
pod = (m.get("pod") or "").strip()
|
|
val = v[1] if isinstance(v, list) and len(v) > 1 else ""
|
|
if pod:
|
|
out.append(f"- restarts({hours}h): {ns}/{pod} = {val}")
|
|
return "\n".join(out)
|
|
|
|
def vm_cluster_snapshot() -> str:
|
|
parts: list[str] = []
|
|
# Node readiness (kube-state-metrics).
|
|
ready = vm_query('sum(kube_node_status_condition{condition="Ready",status="true"})')
|
|
not_ready = vm_query('sum(kube_node_status_condition{condition="Ready",status="false"})')
|
|
if ready and not_ready:
|
|
try:
|
|
r = _vm_value_series(ready)[0]["value"][1]
|
|
nr = _vm_value_series(not_ready)[0]["value"][1]
|
|
parts.append(f"- nodes ready: {r} (not ready: {nr})")
|
|
except Exception:
|
|
pass
|
|
|
|
phases = vm_query("sum by (phase) (kube_pod_status_phase)")
|
|
pr = vm_render_result(phases, limit=8)
|
|
if pr:
|
|
parts.append("Pod phases:")
|
|
parts.append(pr)
|
|
return "\n".join(parts).strip()
|
|
|
|
def _strip_code_fence(text: str) -> str:
|
|
cleaned = (text or "").strip()
|
|
match = CODE_FENCE_RE.match(cleaned)
|
|
if match:
|
|
return match.group(1).strip()
|
|
return cleaned
|
|
|
|
def _normalize_reply(value: Any) -> str:
|
|
if isinstance(value, dict):
|
|
for key in ("content", "response", "reply", "message"):
|
|
if key in value:
|
|
return _normalize_reply(value[key])
|
|
for v in value.values():
|
|
if isinstance(v, (str, dict, list)):
|
|
return _normalize_reply(v)
|
|
return json.dumps(value, ensure_ascii=False)
|
|
if isinstance(value, list):
|
|
parts = [_normalize_reply(item) for item in value]
|
|
return " ".join(p for p in parts if p)
|
|
if value is None:
|
|
return ""
|
|
text = _strip_code_fence(str(value))
|
|
if text.startswith("{") and text.endswith("}"):
|
|
try:
|
|
return _normalize_reply(json.loads(text))
|
|
except Exception:
|
|
return text
|
|
return text
|
|
|
|
|
|
# Internal HTTP endpoint for cluster answers (website uses this).
|
|
class _AtlasbotHandler(BaseHTTPRequestHandler):
|
|
server_version = "AtlasbotHTTP/1.0"
|
|
|
|
def _write_json(self, status: int, payload: dict[str, Any]):
|
|
body = json.dumps(payload).encode("utf-8")
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _authorized(self) -> bool:
|
|
if not ATLASBOT_INTERNAL_TOKEN:
|
|
return True
|
|
token = self.headers.get("X-Internal-Token", "")
|
|
return token == ATLASBOT_INTERNAL_TOKEN
|
|
|
|
def do_GET(self): # noqa: N802
|
|
if self.path == "/health":
|
|
self._write_json(200, {"status": "ok"})
|
|
return
|
|
self._write_json(404, {"error": "not_found"})
|
|
|
|
def do_POST(self): # noqa: N802
|
|
if self.path != "/v1/answer":
|
|
self._write_json(404, {"error": "not_found"})
|
|
return
|
|
if not self._authorized():
|
|
self._write_json(401, {"error": "unauthorized"})
|
|
return
|
|
try:
|
|
length = int(self.headers.get("Content-Length", "0"))
|
|
except ValueError:
|
|
length = 0
|
|
raw = self.rfile.read(length) if length > 0 else b""
|
|
try:
|
|
payload = json.loads(raw.decode("utf-8")) if raw else {}
|
|
except json.JSONDecodeError:
|
|
self._write_json(400, {"error": "invalid_json"})
|
|
return
|
|
prompt = str(payload.get("prompt") or payload.get("question") or "").strip()
|
|
if not prompt:
|
|
self._write_json(400, {"error": "missing_prompt"})
|
|
return
|
|
inventory = node_inventory_live()
|
|
answer = structured_answer(prompt, inventory=inventory, metrics_summary="")
|
|
if not answer and _knowledge_intent(prompt):
|
|
answer = knowledge_summary(prompt, inventory)
|
|
if not answer:
|
|
kb = kb_retrieve_titles(prompt, limit=4)
|
|
answer = kb or ""
|
|
self._write_json(200, {"answer": answer})
|
|
|
|
|
|
def _start_http_server():
|
|
server = HTTPServer(("0.0.0.0", ATLASBOT_HTTP_PORT), _AtlasbotHandler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
|
|
|
|
# Conversation state.
|
|
history = collections.defaultdict(list) # (room_id, sender|None) -> list[str] (short transcript)
|
|
|
|
def key_for(room_id: str, sender: str, is_dm: bool):
|
|
return (room_id, None) if is_dm else (room_id, sender)
|
|
|
|
def build_context(
|
|
prompt: str,
|
|
*,
|
|
allow_tools: bool,
|
|
targets: list[tuple[str, str]],
|
|
inventory: list[dict[str, Any]] | None = None,
|
|
) -> str:
|
|
parts: list[str] = []
|
|
|
|
kb = kb_retrieve(prompt)
|
|
if kb:
|
|
parts.append(kb)
|
|
|
|
endpoints, edges = catalog_hints(prompt)
|
|
if endpoints:
|
|
parts.append(endpoints)
|
|
|
|
node_ctx = node_inventory_context(prompt, inventory)
|
|
if node_ctx:
|
|
parts.append(node_ctx)
|
|
|
|
if allow_tools:
|
|
# Scope pod summaries to relevant namespaces/workloads when possible.
|
|
prefixes_by_ns: dict[str, set[str]] = collections.defaultdict(set)
|
|
for ns, name in (targets or []) + (edges or []):
|
|
if ns and name:
|
|
prefixes_by_ns[ns].add(name)
|
|
pod_lines: list[str] = []
|
|
for ns in sorted(prefixes_by_ns.keys()):
|
|
summary = summarize_pods(ns, prefixes_by_ns[ns])
|
|
if summary:
|
|
pod_lines.append(f"Pods (live):\n{summary}")
|
|
if pod_lines:
|
|
parts.append("\n".join(pod_lines)[:MAX_TOOL_CHARS])
|
|
|
|
flux_bad = flux_not_ready()
|
|
if flux_bad:
|
|
parts.append("Flux (not ready):\n" + flux_bad)
|
|
|
|
p_l = (prompt or "").lower()
|
|
if any(w in p_l for w in METRIC_HINT_WORDS):
|
|
restarts = vm_top_restarts(1)
|
|
if restarts:
|
|
parts.append("VictoriaMetrics (top restarts 1h):\n" + restarts)
|
|
snap = vm_cluster_snapshot()
|
|
if snap:
|
|
parts.append("VictoriaMetrics (cluster snapshot):\n" + snap)
|
|
|
|
return "\n\n".join([p for p in parts if p]).strip()
|
|
|
|
|
|
def _knowledge_intent(prompt: str) -> bool:
|
|
q = normalize_query(prompt)
|
|
return any(
|
|
phrase in q
|
|
for phrase in (
|
|
"what do you know",
|
|
"tell me about",
|
|
"overview",
|
|
"summary",
|
|
"describe",
|
|
"explain",
|
|
"what is",
|
|
)
|
|
)
|
|
|
|
|
|
def _inventory_summary(inventory: list[dict[str, Any]]) -> str:
|
|
if not inventory:
|
|
return ""
|
|
groups = _group_nodes(inventory)
|
|
total = len(inventory)
|
|
ready = [n for n in inventory if n.get("ready") is True]
|
|
not_ready = [n for n in inventory if n.get("ready") is False]
|
|
parts = [f"Atlas cluster: {total} nodes ({len(ready)} ready, {len(not_ready)} not ready)."]
|
|
for key in ("rpi5", "rpi4", "jetson", "amd64", "arm64-unknown", "unknown"):
|
|
nodes = groups.get(key) or []
|
|
if nodes:
|
|
parts.append(f"- {key}: {len(nodes)} nodes ({', '.join(nodes)})")
|
|
return "\n".join(parts)
|
|
|
|
|
|
def knowledge_summary(prompt: str, inventory: list[dict[str, Any]]) -> str:
|
|
parts: list[str] = []
|
|
inv = _inventory_summary(inventory)
|
|
if inv:
|
|
parts.append(inv)
|
|
kb_titles = kb_retrieve_titles(prompt, limit=4)
|
|
if kb_titles:
|
|
parts.append(kb_titles)
|
|
return "\n".join(parts).strip()
|
|
|
|
def _ollama_call(hist_key, prompt: str, *, context: str) -> str:
|
|
system = (
|
|
"System: You are Atlas, the Titan lab assistant for Atlas/Othrys. "
|
|
"Be helpful, direct, and concise. "
|
|
"Prefer answering with exact repo paths and Kubernetes resource names. "
|
|
"Never include or request secret values. "
|
|
"Do not suggest commands unless explicitly asked. "
|
|
"Respond in plain sentences; do not return JSON or code fences unless explicitly asked. "
|
|
"If the answer is not grounded in the provided context or tool data, say you do not know."
|
|
)
|
|
transcript_parts = [system]
|
|
if context:
|
|
transcript_parts.append("Context (grounded):\n" + context[:MAX_KB_CHARS])
|
|
transcript_parts.extend(history[hist_key][-24:])
|
|
transcript_parts.append(f"User: {prompt}")
|
|
transcript = "\n".join(transcript_parts)
|
|
|
|
payload = {"model": MODEL, "message": transcript}
|
|
headers = {"Content-Type": "application/json"}
|
|
if API_KEY:
|
|
headers["x-api-key"] = API_KEY
|
|
r = request.Request(OLLAMA_URL, data=json.dumps(payload).encode(), headers=headers)
|
|
with request.urlopen(r, timeout=OLLAMA_TIMEOUT_SEC) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
raw_reply = data.get("message") or data.get("response") or data.get("reply") or data
|
|
reply = _normalize_reply(raw_reply) or "I'm here to help."
|
|
history[hist_key].append(f"Atlas: {reply}")
|
|
return reply
|
|
|
|
def ollama_reply(hist_key, prompt: str, *, context: str, fallback: str = "") -> str:
|
|
try:
|
|
return _ollama_call(hist_key, prompt, context=context)
|
|
except Exception:
|
|
if fallback:
|
|
history[hist_key].append(f"Atlas: {fallback}")
|
|
return fallback
|
|
return "Model backend is busy. Try again in a moment."
|
|
|
|
def ollama_reply_with_thinking(token: str, room: str, hist_key, prompt: str, *, context: str, fallback: str) -> str:
|
|
result: dict[str, str] = {"reply": ""}
|
|
done = threading.Event()
|
|
|
|
def worker():
|
|
result["reply"] = ollama_reply(hist_key, prompt, context=context, fallback=fallback)
|
|
done.set()
|
|
|
|
thread = threading.Thread(target=worker, daemon=True)
|
|
thread.start()
|
|
if not done.wait(2.0):
|
|
send_msg(token, room, "Thinking…")
|
|
prompt_hint = " ".join((prompt or "").split())
|
|
if len(prompt_hint) > 160:
|
|
prompt_hint = prompt_hint[:157] + "…"
|
|
heartbeat = max(10, THINKING_INTERVAL_SEC)
|
|
next_heartbeat = time.monotonic() + heartbeat
|
|
while not done.wait(max(0, next_heartbeat - time.monotonic())):
|
|
if prompt_hint:
|
|
send_msg(token, room, f"Still thinking about: {prompt_hint} (gathering context)")
|
|
else:
|
|
send_msg(token, room, "Still thinking (gathering context)…")
|
|
next_heartbeat += heartbeat
|
|
thread.join(timeout=1)
|
|
return result["reply"] or fallback or "Model backend is busy. Try again in a moment."
|
|
|
|
def sync_loop(token: str, room_id: str):
|
|
since = None
|
|
try:
|
|
res = req("GET", "/_matrix/client/v3/sync?timeout=0", token, timeout=10)
|
|
since = res.get("next_batch")
|
|
except Exception:
|
|
pass
|
|
|
|
while True:
|
|
params = {"timeout": 30000}
|
|
if since:
|
|
params["since"] = since
|
|
query = parse.urlencode(params)
|
|
try:
|
|
res = req("GET", f"/_matrix/client/v3/sync?{query}", token, timeout=35)
|
|
except Exception:
|
|
time.sleep(5)
|
|
continue
|
|
since = res.get("next_batch", since)
|
|
|
|
# invites
|
|
for rid, data in res.get("rooms", {}).get("invite", {}).items():
|
|
try:
|
|
join_room(token, rid)
|
|
except Exception:
|
|
pass
|
|
|
|
# messages
|
|
for rid, data in res.get("rooms", {}).get("join", {}).items():
|
|
timeline = data.get("timeline", {}).get("events", [])
|
|
joined_count = data.get("summary", {}).get("m.joined_member_count")
|
|
is_dm = joined_count is not None and joined_count <= 2
|
|
|
|
for ev in timeline:
|
|
if ev.get("type") != "m.room.message":
|
|
continue
|
|
content = ev.get("content", {})
|
|
body = (content.get("body", "") or "").strip()
|
|
if not body:
|
|
continue
|
|
sender = ev.get("sender", "")
|
|
if sender == f"@{USER}:live.bstein.dev":
|
|
continue
|
|
|
|
mentioned = is_mentioned(content, body)
|
|
hist_key = key_for(rid, sender, is_dm)
|
|
history[hist_key].append(f"{sender}: {body}")
|
|
history[hist_key] = history[hist_key][-80:]
|
|
|
|
if not (is_dm or mentioned):
|
|
continue
|
|
|
|
lower_body = body.lower()
|
|
|
|
# Only do live cluster introspection in DMs; metrics can be answered when mentioned.
|
|
allow_tools = is_dm
|
|
allow_metrics = is_dm or mentioned
|
|
|
|
promql = ""
|
|
if allow_tools:
|
|
m = re.match(r"(?is)^\\s*promql\\s*(?:\\:|\\s)\\s*(.+?)\\s*$", body)
|
|
if m:
|
|
promql = m.group(1).strip()
|
|
|
|
# Attempt to scope tools to the most likely workloads when hostnames are mentioned.
|
|
targets: list[tuple[str, str]] = []
|
|
for m in HOST_RE.finditer(body.lower()):
|
|
host = m.group(1).lower()
|
|
for ep in _HOST_INDEX.get(host, []):
|
|
backend = ep.get("backend") or {}
|
|
ns = backend.get("namespace") or ""
|
|
for w in backend.get("workloads") or []:
|
|
if isinstance(w, dict) and w.get("name"):
|
|
targets.append((ns, str(w["name"])))
|
|
|
|
inventory = node_inventory_for_prompt(body)
|
|
context = build_context(body, allow_tools=allow_tools, targets=targets, inventory=inventory)
|
|
if allow_tools and promql:
|
|
res = vm_query(promql, timeout=20)
|
|
rendered = vm_render_result(res, limit=15) or "(no results)"
|
|
extra = "VictoriaMetrics (PromQL result):\n" + rendered
|
|
context = (context + "\n\n" + extra).strip() if context else extra
|
|
metrics_context, metrics_fallback = metrics_query_context(body, allow_tools=allow_metrics)
|
|
if metrics_context:
|
|
context = (context + "\n\n" + metrics_context).strip() if context else metrics_context
|
|
|
|
fallback = metrics_fallback or ""
|
|
if not fallback and context:
|
|
fallback = _context_fallback(context)
|
|
|
|
structured = structured_answer(body, inventory=inventory, metrics_summary=metrics_fallback or "")
|
|
if structured:
|
|
send_msg(token, rid, structured)
|
|
continue
|
|
|
|
if _knowledge_intent(body):
|
|
summary = knowledge_summary(body, inventory)
|
|
if summary:
|
|
send_msg(token, rid, summary)
|
|
continue
|
|
|
|
reply = ollama_reply_with_thinking(
|
|
token,
|
|
rid,
|
|
hist_key,
|
|
body,
|
|
context=context,
|
|
fallback=fallback,
|
|
)
|
|
send_msg(token, rid, reply)
|
|
|
|
def login_with_retry():
|
|
last_err = None
|
|
for attempt in range(10):
|
|
try:
|
|
return login()
|
|
except Exception as exc: # noqa: BLE001
|
|
last_err = exc
|
|
time.sleep(min(30, 2 ** attempt))
|
|
raise last_err
|
|
|
|
def main():
|
|
load_kb()
|
|
_start_http_server()
|
|
token = login_with_retry()
|
|
try:
|
|
room_id = resolve_alias(token, ROOM_ALIAS)
|
|
join_room(token, room_id)
|
|
except Exception:
|
|
room_id = None
|
|
sync_loop(token, room_id)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|