atlasbot: broaden intent routing and retriever selection
This commit is contained in:
parent
c356abdec0
commit
d85ac70c83
@ -110,6 +110,8 @@ class ScoreContext:
|
|||||||
sub_questions: list[str]
|
sub_questions: list[str]
|
||||||
retries: int
|
retries: int
|
||||||
parallelism: int
|
parallelism: int
|
||||||
|
select_best: bool
|
||||||
|
fast_model: str
|
||||||
|
|
||||||
|
|
||||||
class AnswerEngine:
|
class AnswerEngine:
|
||||||
@ -155,6 +157,7 @@ class AnswerEngine:
|
|||||||
"route",
|
"route",
|
||||||
"decompose",
|
"decompose",
|
||||||
"chunk_score",
|
"chunk_score",
|
||||||
|
"chunk_select",
|
||||||
"fact_select",
|
"fact_select",
|
||||||
"synth",
|
"synth",
|
||||||
"subanswer",
|
"subanswer",
|
||||||
@ -265,7 +268,7 @@ class AnswerEngine:
|
|||||||
if spine_line:
|
if spine_line:
|
||||||
key_facts = _merge_fact_lines([spine_line], key_facts)
|
key_facts = _merge_fact_lines([spine_line], key_facts)
|
||||||
metric_facts = _merge_fact_lines([spine_line], metric_facts)
|
metric_facts = _merge_fact_lines([spine_line], metric_facts)
|
||||||
if spine_answer and mode == "fast":
|
if spine_answer and mode in {"fast", "quick"}:
|
||||||
scores = _default_scores()
|
scores = _default_scores()
|
||||||
meta = _build_meta(mode, call_count, call_cap, limit_hit, classify, tool_hint, started)
|
meta = _build_meta(mode, call_count, call_cap, limit_hit, classify, tool_hint, started)
|
||||||
return AnswerResult(spine_answer, scores, meta)
|
return AnswerResult(spine_answer, scores, meta)
|
||||||
@ -1343,6 +1346,8 @@ async def _score_chunks(
|
|||||||
sub_questions=sub_questions,
|
sub_questions=sub_questions,
|
||||||
retries=max(1, plan.score_retries),
|
retries=max(1, plan.score_retries),
|
||||||
parallelism=plan.parallelism,
|
parallelism=plan.parallelism,
|
||||||
|
select_best=plan.score_retries > 1,
|
||||||
|
fast_model=plan.fast_model,
|
||||||
)
|
)
|
||||||
if ctx.parallelism <= 1 or len(groups) * ctx.retries <= 1:
|
if ctx.parallelism <= 1 or len(groups) * ctx.retries <= 1:
|
||||||
return await _score_groups_serial(call_llm, groups, ctx)
|
return await _score_groups_serial(call_llm, groups, ctx)
|
||||||
@ -1357,6 +1362,10 @@ async def _score_groups_serial(
|
|||||||
scores: dict[str, float] = {}
|
scores: dict[str, float] = {}
|
||||||
for grp in groups:
|
for grp in groups:
|
||||||
runs = [await _score_chunk_group(call_llm, grp, ctx.question, ctx.sub_questions) for _ in range(ctx.retries)]
|
runs = [await _score_chunk_group(call_llm, grp, ctx.question, ctx.sub_questions) for _ in range(ctx.retries)]
|
||||||
|
if ctx.select_best and len(runs) > 1:
|
||||||
|
best = await _select_best_score_run(call_llm, grp, runs, ctx)
|
||||||
|
scores.update(best)
|
||||||
|
else:
|
||||||
scores.update(_merge_score_runs(runs))
|
scores.update(_merge_score_runs(runs))
|
||||||
return scores
|
return scores
|
||||||
|
|
||||||
@ -1375,7 +1384,12 @@ async def _score_groups_parallel(
|
|||||||
for idx, result in results:
|
for idx, result in results:
|
||||||
grouped.setdefault(idx, []).append(result)
|
grouped.setdefault(idx, []).append(result)
|
||||||
scores: dict[str, float] = {}
|
scores: dict[str, float] = {}
|
||||||
for runs in grouped.values():
|
for idx, runs in grouped.items():
|
||||||
|
if ctx.select_best and len(runs) > 1:
|
||||||
|
group = groups[idx]
|
||||||
|
best = await _select_best_score_run(call_llm, group, runs, ctx)
|
||||||
|
scores.update(best)
|
||||||
|
else:
|
||||||
scores.update(_merge_score_runs(runs))
|
scores.update(_merge_score_runs(runs))
|
||||||
return scores
|
return scores
|
||||||
|
|
||||||
@ -1434,6 +1448,38 @@ def _merge_score_runs(runs: list[dict[str, float]]) -> dict[str, float]:
|
|||||||
return {key: totals[key] / counts[key] for key in totals}
|
return {key: totals[key] / counts[key] for key in totals}
|
||||||
|
|
||||||
|
|
||||||
|
async def _select_best_score_run(
|
||||||
|
call_llm: Callable[..., Any],
|
||||||
|
group: list[dict[str, Any]],
|
||||||
|
runs: list[dict[str, float]],
|
||||||
|
ctx: ScoreContext,
|
||||||
|
) -> dict[str, float]:
|
||||||
|
if not runs:
|
||||||
|
return {}
|
||||||
|
prompt = (
|
||||||
|
prompts.RETRIEVER_SELECT_PROMPT
|
||||||
|
+ "\nQuestion: "
|
||||||
|
+ ctx.question
|
||||||
|
+ "\nSubQuestions: "
|
||||||
|
+ json.dumps(ctx.sub_questions)
|
||||||
|
+ "\nChunks: "
|
||||||
|
+ json.dumps(group)
|
||||||
|
+ "\nRuns: "
|
||||||
|
+ json.dumps(runs)
|
||||||
|
)
|
||||||
|
raw = await call_llm(prompts.RETRIEVER_SELECT_SYSTEM, prompt, model=ctx.fast_model, tag="chunk_select")
|
||||||
|
data = parse_json(raw)
|
||||||
|
idx = 0
|
||||||
|
if isinstance(data, dict):
|
||||||
|
try:
|
||||||
|
idx = int(data.get("selected_index") or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
idx = 0
|
||||||
|
if idx < 0 or idx >= len(runs):
|
||||||
|
idx = 0
|
||||||
|
return runs[idx]
|
||||||
|
|
||||||
|
|
||||||
def _keyword_hits(
|
def _keyword_hits(
|
||||||
ranked: list[dict[str, Any]],
|
ranked: list[dict[str, Any]],
|
||||||
head: dict[str, Any],
|
head: dict[str, Any],
|
||||||
@ -1574,7 +1620,16 @@ def _spine_lines(lines: list[str]) -> dict[str, str]:
|
|||||||
if nodes_line:
|
if nodes_line:
|
||||||
spine["nodes_count"] = nodes_line
|
spine["nodes_count"] = nodes_line
|
||||||
spine["nodes_ready"] = nodes_line
|
spine["nodes_ready"] = nodes_line
|
||||||
|
else:
|
||||||
|
nodes_total = _line_starting_with(lines, "nodes_total:")
|
||||||
|
nodes_ready = _line_starting_with(lines, "nodes_ready:")
|
||||||
|
if nodes_total:
|
||||||
|
spine["nodes_count"] = nodes_total
|
||||||
|
if nodes_ready:
|
||||||
|
spine["nodes_ready"] = nodes_ready
|
||||||
hardware_line = _line_starting_with(lines, "hardware_nodes:")
|
hardware_line = _line_starting_with(lines, "hardware_nodes:")
|
||||||
|
if not hardware_line:
|
||||||
|
hardware_line = _line_starting_with(lines, "hardware:")
|
||||||
if hardware_line:
|
if hardware_line:
|
||||||
spine["nodes_non_rpi"] = hardware_line
|
spine["nodes_non_rpi"] = hardware_line
|
||||||
hottest_line = _line_starting_with(lines, "hottest:")
|
hottest_line = _line_starting_with(lines, "hottest:")
|
||||||
@ -1613,7 +1668,14 @@ def _parse_group_line(line: str) -> dict[str, list[str]]:
|
|||||||
if not part or "=" not in part:
|
if not part or "=" not in part:
|
||||||
continue
|
continue
|
||||||
key, value = part.split("=", 1)
|
key, value = part.split("=", 1)
|
||||||
nodes = [item.strip() for item in value.split(",") if item.strip()]
|
value = value.strip()
|
||||||
|
nodes: list[str] = []
|
||||||
|
if "(" in value and ")" in value:
|
||||||
|
inner = value[value.find("(") + 1 : value.rfind(")")]
|
||||||
|
nodes = [item.strip() for item in inner.split(",") if item.strip()]
|
||||||
|
if not nodes:
|
||||||
|
cleaned = re.sub(r"^[0-9]+", "", value).strip()
|
||||||
|
nodes = [item.strip() for item in cleaned.split(",") if item.strip()]
|
||||||
groups[key.strip()] = nodes
|
groups[key.strip()] = nodes
|
||||||
return groups
|
return groups
|
||||||
|
|
||||||
|
|||||||
@ -10,22 +10,22 @@ class IntentMatch:
|
|||||||
score: int
|
score: int
|
||||||
|
|
||||||
|
|
||||||
_COUNT_TERMS = r"(how many|count|number of|total|totals|tally|amount of|quantity|sum of|overall|in total|all up)"
|
_COUNT_TERMS = r"(how\\s+many|count|number\\s+of|total|totals|tally|amount\\s+of|quantity|sum\\s+of|overall|in\\s+total|all\\s+up)"
|
||||||
_NODE_TERMS = r"(nodes?|workers?|worker nodes?|cluster nodes?|machines?|hosts?|members?|instances?)"
|
_NODE_TERMS = r"(nodes?|workers?|worker\\s+nodes?|cluster\\s+nodes?|machines?|hosts?|members?|instances?|servers?|agents?|control[-\\s]?plane|control\\s+plane)"
|
||||||
_READY_TERMS = r"(ready|unready|not ready|down|offline|not responding|missing)"
|
_READY_TERMS = r"(ready|unready|not\\s+ready|down|offline|not\\s+responding|missing|lost|gone|drain(?:ed|ing)?|cordon(?:ed|ing)?)"
|
||||||
_HOTTEST_TERMS = r"(hottest|hot|highest|max(?:imum)?|peak|top|most|worst|spikiest|heaviest|largest)"
|
_HOTTEST_TERMS = r"(hottest|hot|highest|max(?:imum)?|peak|top|most|worst|spikiest|heaviest|largest|biggest|noisiest|loudest)"
|
||||||
_CPU_TERMS = r"(cpu|processor|compute|core|cores|load|load avg|load average)"
|
_CPU_TERMS = r"(cpu|processor|processors|compute|core|cores|load|load\\s+avg|load\\s+average|util(?:ization)?|usage)"
|
||||||
_RAM_TERMS = r"(ram|memory|mem|heap)"
|
_RAM_TERMS = r"(ram|memory|mem|heap|rss|resident|swap)"
|
||||||
_NET_TERMS = r"(net|network|bandwidth|throughput|traffic|rx|tx|ingress|egress|bits|bytes)"
|
_NET_TERMS = r"(net|network|bandwidth|throughput|traffic|rx|tx|ingress|egress|bits|bytes|packets|pps|bps)"
|
||||||
_IO_TERMS = r"(io|i/o|disk io|disk activity|read/write|read write|storage io|iops)"
|
_IO_TERMS = r"(io|i/o|disk\\s+io|disk\\s+activity|read/?write|storage\\s+io|iops|latency)"
|
||||||
_DISK_TERMS = r"(disk|storage|volume|pvc|filesystem|fs|capacity|space)"
|
_DISK_TERMS = r"(disk|storage|volume|pvc|filesystem|fs|capacity|space|full|usage)"
|
||||||
_PG_TERMS = r"(postgres|postgresql|pg\\b|database|db|sql)"
|
_PG_TERMS = r"(postgres|postgresql|pg\\b|database|db|sql|psql)"
|
||||||
_CONN_TERMS = r"(connections?|conn|pool|sessions?|clients?)"
|
_CONN_TERMS = r"(connections?|conn|pool|sessions?|clients?|active\\s+connections?|open\\s+connections?)"
|
||||||
_DB_HOT_TERMS = r"(hottest|busiest|most|largest|top|heaviest)"
|
_DB_HOT_TERMS = r"(hottest|busiest|most|largest|top|heaviest|noisiest|highest\\s+load)"
|
||||||
_NAMESPACE_TERMS = r"(namespace|namespaces|ns\\b)"
|
_NAMESPACE_TERMS = r"(namespace|namespaces|ns\\b|tenant|workload\\s+namespace)"
|
||||||
_PODS_TERMS = r"(pods?|workloads?|tasks?|containers?)"
|
_PODS_TERMS = r"(pods?|workloads?|tasks?|containers?|deployments?|jobs?|cronjobs?|daemonsets?|statefulsets?)"
|
||||||
_NON_RPI_TERMS = r"(non[-\\s]?raspberry|not\\s+raspberry|non[-\\s]?rpi|not\\s+rpi|amd64|x86|x86_64|jetson)"
|
_NON_RPI_TERMS = r"(non[-\\s]?raspberry|not\\s+raspberry|non[-\\s]?rpi|not\\s+rpi|amd64|x86|x86_64|intel|ryzen|jetson|arm64\\b(?!.*rpi))"
|
||||||
_PRESSURE_TERMS = r"(pressure|overload|hotspot|bottleneck|saturation|headroom|strain|stress)"
|
_PRESSURE_TERMS = r"(pressure|overload|hotspot|bottleneck|saturation|headroom|strain|stress|critical|warning|at\\s+capacity|near\\s+limit)"
|
||||||
|
|
||||||
|
|
||||||
def route_intent(question: str) -> IntentMatch | None:
|
def route_intent(question: str) -> IntentMatch | None:
|
||||||
@ -33,11 +33,11 @@ def route_intent(question: str) -> IntentMatch | None:
|
|||||||
if not text:
|
if not text:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if re.search(_COUNT_TERMS, text) and re.search(_NODE_TERMS, text):
|
if re.search(_COUNT_TERMS, text) and (re.search(_NODE_TERMS, text) or "cluster" in text):
|
||||||
return IntentMatch("nodes_count", 90)
|
return IntentMatch("nodes_count", 90)
|
||||||
if re.search(_READY_TERMS, text) and re.search(_NODE_TERMS, text):
|
if re.search(_READY_TERMS, text) and (re.search(_NODE_TERMS, text) or "cluster" in text or "workers" in text):
|
||||||
return IntentMatch("nodes_ready", 85)
|
return IntentMatch("nodes_ready", 85)
|
||||||
if re.search(_NON_RPI_TERMS, text) and re.search(_NODE_TERMS, text):
|
if re.search(_NON_RPI_TERMS, text) and (re.search(_NODE_TERMS, text) or "cluster" in text):
|
||||||
return IntentMatch("nodes_non_rpi", 80)
|
return IntentMatch("nodes_non_rpi", 80)
|
||||||
|
|
||||||
if re.search(_HOTTEST_TERMS, text) and re.search(_CPU_TERMS, text):
|
if re.search(_HOTTEST_TERMS, text) and re.search(_CPU_TERMS, text):
|
||||||
|
|||||||
@ -59,6 +59,18 @@ CHUNK_SCORE_PROMPT = (
|
|||||||
"Return JSON list of objects with: id, score, reason (<=12 words)."
|
"Return JSON list of objects with: id, score, reason (<=12 words)."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
RETRIEVER_SELECT_SYSTEM = (
|
||||||
|
CLUSTER_SYSTEM
|
||||||
|
+ " Select the best relevance scoring run for the chunk group. "
|
||||||
|
+ "Return JSON only."
|
||||||
|
)
|
||||||
|
|
||||||
|
RETRIEVER_SELECT_PROMPT = (
|
||||||
|
"You are given multiple scoring runs for the same chunk group. "
|
||||||
|
"Choose the run that best aligns with the question and sub-questions. "
|
||||||
|
"Return JSON with fields: selected_index (int), rationale (<=16 words)."
|
||||||
|
)
|
||||||
|
|
||||||
METRIC_PREFIX_SYSTEM = (
|
METRIC_PREFIX_SYSTEM = (
|
||||||
CLUSTER_SYSTEM
|
CLUSTER_SYSTEM
|
||||||
+ " Select relevant metric prefixes from the available list. "
|
+ " Select relevant metric prefixes from the available list. "
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user