atlasbot: make smart/genius conclude via fact-sheet path
This commit is contained in:
parent
24e0a342cd
commit
01b27d5488
@ -248,13 +248,31 @@ class AnswerEngine:
|
||||
classify: dict[str, Any] = {}
|
||||
tool_hint: dict[str, Any] | None = None
|
||||
try:
|
||||
if mode in {"quick", "fast"} and not limitless:
|
||||
if mode in {"quick", "fast", "smart", "genius"} and not limitless:
|
||||
if observer:
|
||||
observer("factsheet", "building fact sheet")
|
||||
if _is_plain_math_question(question):
|
||||
reply = (
|
||||
"I focus on Titan cluster operations. Ask me about cluster health, nodes, workloads, "
|
||||
"namespaces, storage, or alerts."
|
||||
)
|
||||
scores = _default_scores()
|
||||
meta = _build_meta(
|
||||
mode,
|
||||
call_count,
|
||||
call_cap,
|
||||
limit_hit,
|
||||
time_budget_hit,
|
||||
time_budget_sec,
|
||||
classify,
|
||||
tool_hint,
|
||||
started,
|
||||
)
|
||||
return AnswerResult(reply, scores, meta)
|
||||
kb_lines = (
|
||||
self._kb.chunk_lines(
|
||||
max_files=plan.kb_max_files,
|
||||
max_chars=max(1200, plan.kb_max_chars),
|
||||
max_chars=_factsheet_kb_chars(mode, plan.kb_max_chars),
|
||||
)
|
||||
if self._kb
|
||||
else []
|
||||
@ -263,15 +281,15 @@ class AnswerEngine:
|
||||
question,
|
||||
summary_lines,
|
||||
kb_lines,
|
||||
limit=max(14, plan.chunk_top * 5),
|
||||
limit=_factsheet_line_limit(mode),
|
||||
)
|
||||
if observer:
|
||||
observer("quick", "answering from fact sheet")
|
||||
classify = {
|
||||
"needs_snapshot": True,
|
||||
"needs_kb": bool(kb_lines),
|
||||
"question_type": "quick_factsheet",
|
||||
"answer_style": "direct",
|
||||
"question_type": f"{mode}_factsheet",
|
||||
"answer_style": "direct" if mode in {"quick", "fast"} else "concise",
|
||||
"follow_up": False,
|
||||
}
|
||||
heuristic_reply = _quick_fact_sheet_heuristic_answer(question, fact_lines)
|
||||
@ -294,15 +312,15 @@ class AnswerEngine:
|
||||
quick_prompt = (
|
||||
"Question: "
|
||||
+ question
|
||||
+ "\nAnswer using only the Fact Sheet. Keep it to 1-3 sentences. "
|
||||
+ "If the Fact Sheet is missing key data, say exactly what is missing and suggest atlas-smart."
|
||||
+ "\nAnswer using only the Fact Sheet. "
|
||||
+ _factsheet_instruction(mode)
|
||||
)
|
||||
reply = await call_llm(
|
||||
prompts.ANSWER_SYSTEM,
|
||||
quick_prompt,
|
||||
context=quick_context,
|
||||
model=plan.fast_model,
|
||||
tag="quick_factsheet",
|
||||
model=_factsheet_model(mode, plan),
|
||||
tag=f"{mode}_factsheet",
|
||||
)
|
||||
reply = _strip_followup_meta(reply)
|
||||
scores = _default_scores()
|
||||
@ -3386,6 +3404,73 @@ def _state_from_payload(payload: dict[str, Any] | None) -> ConversationState | N
|
||||
)
|
||||
|
||||
|
||||
def _factsheet_kb_chars(mode: str, default_chars: int) -> int:
|
||||
if mode == "genius":
|
||||
return min(max(default_chars, 4000), 6000)
|
||||
if mode == "smart":
|
||||
return min(max(default_chars, 3000), 4500)
|
||||
return max(1200, default_chars)
|
||||
|
||||
|
||||
def _factsheet_line_limit(mode: str) -> int:
|
||||
if mode == "genius":
|
||||
return 30
|
||||
if mode == "smart":
|
||||
return 22
|
||||
return 14
|
||||
|
||||
|
||||
def _factsheet_instruction(mode: str) -> str:
|
||||
if mode == "genius":
|
||||
return (
|
||||
"Start with a direct conclusion, then include the strongest supporting facts and one caveat. "
|
||||
"Keep it to 4-8 sentences. If data is missing, name the missing metric explicitly."
|
||||
)
|
||||
if mode == "smart":
|
||||
return (
|
||||
"Start with a direct conclusion and support it with key facts. Keep it to 2-5 sentences. "
|
||||
"If data is missing, say exactly what is missing and suggest atlas-genius."
|
||||
)
|
||||
return "Keep it to 1-3 sentences. If key data is missing, say what is missing and suggest atlas-smart."
|
||||
|
||||
|
||||
def _factsheet_model(mode: str, plan: ModePlan) -> str:
|
||||
if mode in {"quick", "fast"}:
|
||||
return plan.fast_model
|
||||
return plan.model
|
||||
|
||||
|
||||
def _is_plain_math_question(question: str) -> bool:
|
||||
lowered = question.lower().strip()
|
||||
if not lowered:
|
||||
return False
|
||||
cluster_markers = (
|
||||
"titan",
|
||||
"atlas",
|
||||
"cluster",
|
||||
"node",
|
||||
"pod",
|
||||
"namespace",
|
||||
"workload",
|
||||
"grafana",
|
||||
"alert",
|
||||
"k8s",
|
||||
"kubernetes",
|
||||
"rpi",
|
||||
"longhorn",
|
||||
"postgres",
|
||||
"victoria",
|
||||
"ollama",
|
||||
)
|
||||
if any(token in lowered for token in cluster_markers):
|
||||
return False
|
||||
if re.fullmatch(r"[0-9\s+\-*/().=]+", lowered):
|
||||
return True
|
||||
if re.search(r"\bwhat(?:'s| is)\s+\d+\s*[-+*/]\s*\d+\b", lowered):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _quick_fact_sheet_lines(
|
||||
question: str,
|
||||
summary_lines: list[str],
|
||||
|
||||
@ -101,6 +101,41 @@ def test_engine_answer_basic():
|
||||
assert "Atlas has 22 nodes" in result.reply
|
||||
|
||||
|
||||
def test_smart_mode_uses_factsheet_path():
|
||||
llm = FakeLLM()
|
||||
settings = _settings()
|
||||
kb = KnowledgeBase("")
|
||||
snapshot = SnapshotProvider(settings)
|
||||
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||
|
||||
result = asyncio.run(engine.answer("What is the most demanding system in titan lab currently?", mode="smart"))
|
||||
assert "Atlas has 22 nodes" in result.reply
|
||||
assert "time budget" not in result.reply.lower()
|
||||
|
||||
|
||||
def test_genius_mode_uses_factsheet_path():
|
||||
llm = FakeLLM()
|
||||
settings = _settings()
|
||||
kb = KnowledgeBase("")
|
||||
snapshot = SnapshotProvider(settings)
|
||||
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||
|
||||
result = asyncio.run(engine.answer("What is the most demanding system in titan lab currently?", mode="genius"))
|
||||
assert "Atlas has 22 nodes" in result.reply
|
||||
assert "time budget" not in result.reply.lower()
|
||||
|
||||
|
||||
def test_plain_math_question_is_rejected_for_cluster_modes():
|
||||
llm = FakeLLM()
|
||||
settings = _settings()
|
||||
kb = KnowledgeBase("")
|
||||
snapshot = SnapshotProvider(settings)
|
||||
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||
|
||||
result = asyncio.run(engine.answer("what is 2+2?", mode="quick"))
|
||||
assert "focus on Titan cluster operations" in result.reply
|
||||
|
||||
|
||||
def test_quick_mode_time_budget_guard():
|
||||
llm = SlowFakeLLM()
|
||||
settings = replace(_settings(), quick_time_budget_sec=0.01)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user