atlasbot: make smart/genius conclude via fact-sheet path
This commit is contained in:
parent
24e0a342cd
commit
01b27d5488
@ -248,13 +248,31 @@ class AnswerEngine:
|
|||||||
classify: dict[str, Any] = {}
|
classify: dict[str, Any] = {}
|
||||||
tool_hint: dict[str, Any] | None = None
|
tool_hint: dict[str, Any] | None = None
|
||||||
try:
|
try:
|
||||||
if mode in {"quick", "fast"} and not limitless:
|
if mode in {"quick", "fast", "smart", "genius"} and not limitless:
|
||||||
if observer:
|
if observer:
|
||||||
observer("factsheet", "building fact sheet")
|
observer("factsheet", "building fact sheet")
|
||||||
|
if _is_plain_math_question(question):
|
||||||
|
reply = (
|
||||||
|
"I focus on Titan cluster operations. Ask me about cluster health, nodes, workloads, "
|
||||||
|
"namespaces, storage, or alerts."
|
||||||
|
)
|
||||||
|
scores = _default_scores()
|
||||||
|
meta = _build_meta(
|
||||||
|
mode,
|
||||||
|
call_count,
|
||||||
|
call_cap,
|
||||||
|
limit_hit,
|
||||||
|
time_budget_hit,
|
||||||
|
time_budget_sec,
|
||||||
|
classify,
|
||||||
|
tool_hint,
|
||||||
|
started,
|
||||||
|
)
|
||||||
|
return AnswerResult(reply, scores, meta)
|
||||||
kb_lines = (
|
kb_lines = (
|
||||||
self._kb.chunk_lines(
|
self._kb.chunk_lines(
|
||||||
max_files=plan.kb_max_files,
|
max_files=plan.kb_max_files,
|
||||||
max_chars=max(1200, plan.kb_max_chars),
|
max_chars=_factsheet_kb_chars(mode, plan.kb_max_chars),
|
||||||
)
|
)
|
||||||
if self._kb
|
if self._kb
|
||||||
else []
|
else []
|
||||||
@ -263,15 +281,15 @@ class AnswerEngine:
|
|||||||
question,
|
question,
|
||||||
summary_lines,
|
summary_lines,
|
||||||
kb_lines,
|
kb_lines,
|
||||||
limit=max(14, plan.chunk_top * 5),
|
limit=_factsheet_line_limit(mode),
|
||||||
)
|
)
|
||||||
if observer:
|
if observer:
|
||||||
observer("quick", "answering from fact sheet")
|
observer("quick", "answering from fact sheet")
|
||||||
classify = {
|
classify = {
|
||||||
"needs_snapshot": True,
|
"needs_snapshot": True,
|
||||||
"needs_kb": bool(kb_lines),
|
"needs_kb": bool(kb_lines),
|
||||||
"question_type": "quick_factsheet",
|
"question_type": f"{mode}_factsheet",
|
||||||
"answer_style": "direct",
|
"answer_style": "direct" if mode in {"quick", "fast"} else "concise",
|
||||||
"follow_up": False,
|
"follow_up": False,
|
||||||
}
|
}
|
||||||
heuristic_reply = _quick_fact_sheet_heuristic_answer(question, fact_lines)
|
heuristic_reply = _quick_fact_sheet_heuristic_answer(question, fact_lines)
|
||||||
@ -294,15 +312,15 @@ class AnswerEngine:
|
|||||||
quick_prompt = (
|
quick_prompt = (
|
||||||
"Question: "
|
"Question: "
|
||||||
+ question
|
+ question
|
||||||
+ "\nAnswer using only the Fact Sheet. Keep it to 1-3 sentences. "
|
+ "\nAnswer using only the Fact Sheet. "
|
||||||
+ "If the Fact Sheet is missing key data, say exactly what is missing and suggest atlas-smart."
|
+ _factsheet_instruction(mode)
|
||||||
)
|
)
|
||||||
reply = await call_llm(
|
reply = await call_llm(
|
||||||
prompts.ANSWER_SYSTEM,
|
prompts.ANSWER_SYSTEM,
|
||||||
quick_prompt,
|
quick_prompt,
|
||||||
context=quick_context,
|
context=quick_context,
|
||||||
model=plan.fast_model,
|
model=_factsheet_model(mode, plan),
|
||||||
tag="quick_factsheet",
|
tag=f"{mode}_factsheet",
|
||||||
)
|
)
|
||||||
reply = _strip_followup_meta(reply)
|
reply = _strip_followup_meta(reply)
|
||||||
scores = _default_scores()
|
scores = _default_scores()
|
||||||
@ -3386,6 +3404,73 @@ def _state_from_payload(payload: dict[str, Any] | None) -> ConversationState | N
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _factsheet_kb_chars(mode: str, default_chars: int) -> int:
|
||||||
|
if mode == "genius":
|
||||||
|
return min(max(default_chars, 4000), 6000)
|
||||||
|
if mode == "smart":
|
||||||
|
return min(max(default_chars, 3000), 4500)
|
||||||
|
return max(1200, default_chars)
|
||||||
|
|
||||||
|
|
||||||
|
def _factsheet_line_limit(mode: str) -> int:
|
||||||
|
if mode == "genius":
|
||||||
|
return 30
|
||||||
|
if mode == "smart":
|
||||||
|
return 22
|
||||||
|
return 14
|
||||||
|
|
||||||
|
|
||||||
|
def _factsheet_instruction(mode: str) -> str:
|
||||||
|
if mode == "genius":
|
||||||
|
return (
|
||||||
|
"Start with a direct conclusion, then include the strongest supporting facts and one caveat. "
|
||||||
|
"Keep it to 4-8 sentences. If data is missing, name the missing metric explicitly."
|
||||||
|
)
|
||||||
|
if mode == "smart":
|
||||||
|
return (
|
||||||
|
"Start with a direct conclusion and support it with key facts. Keep it to 2-5 sentences. "
|
||||||
|
"If data is missing, say exactly what is missing and suggest atlas-genius."
|
||||||
|
)
|
||||||
|
return "Keep it to 1-3 sentences. If key data is missing, say what is missing and suggest atlas-smart."
|
||||||
|
|
||||||
|
|
||||||
|
def _factsheet_model(mode: str, plan: ModePlan) -> str:
|
||||||
|
if mode in {"quick", "fast"}:
|
||||||
|
return plan.fast_model
|
||||||
|
return plan.model
|
||||||
|
|
||||||
|
|
||||||
|
def _is_plain_math_question(question: str) -> bool:
|
||||||
|
lowered = question.lower().strip()
|
||||||
|
if not lowered:
|
||||||
|
return False
|
||||||
|
cluster_markers = (
|
||||||
|
"titan",
|
||||||
|
"atlas",
|
||||||
|
"cluster",
|
||||||
|
"node",
|
||||||
|
"pod",
|
||||||
|
"namespace",
|
||||||
|
"workload",
|
||||||
|
"grafana",
|
||||||
|
"alert",
|
||||||
|
"k8s",
|
||||||
|
"kubernetes",
|
||||||
|
"rpi",
|
||||||
|
"longhorn",
|
||||||
|
"postgres",
|
||||||
|
"victoria",
|
||||||
|
"ollama",
|
||||||
|
)
|
||||||
|
if any(token in lowered for token in cluster_markers):
|
||||||
|
return False
|
||||||
|
if re.fullmatch(r"[0-9\s+\-*/().=]+", lowered):
|
||||||
|
return True
|
||||||
|
if re.search(r"\bwhat(?:'s| is)\s+\d+\s*[-+*/]\s*\d+\b", lowered):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _quick_fact_sheet_lines(
|
def _quick_fact_sheet_lines(
|
||||||
question: str,
|
question: str,
|
||||||
summary_lines: list[str],
|
summary_lines: list[str],
|
||||||
|
|||||||
@ -101,6 +101,41 @@ def test_engine_answer_basic():
|
|||||||
assert "Atlas has 22 nodes" in result.reply
|
assert "Atlas has 22 nodes" in result.reply
|
||||||
|
|
||||||
|
|
||||||
|
def test_smart_mode_uses_factsheet_path():
|
||||||
|
llm = FakeLLM()
|
||||||
|
settings = _settings()
|
||||||
|
kb = KnowledgeBase("")
|
||||||
|
snapshot = SnapshotProvider(settings)
|
||||||
|
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||||
|
|
||||||
|
result = asyncio.run(engine.answer("What is the most demanding system in titan lab currently?", mode="smart"))
|
||||||
|
assert "Atlas has 22 nodes" in result.reply
|
||||||
|
assert "time budget" not in result.reply.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_genius_mode_uses_factsheet_path():
|
||||||
|
llm = FakeLLM()
|
||||||
|
settings = _settings()
|
||||||
|
kb = KnowledgeBase("")
|
||||||
|
snapshot = SnapshotProvider(settings)
|
||||||
|
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||||
|
|
||||||
|
result = asyncio.run(engine.answer("What is the most demanding system in titan lab currently?", mode="genius"))
|
||||||
|
assert "Atlas has 22 nodes" in result.reply
|
||||||
|
assert "time budget" not in result.reply.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_math_question_is_rejected_for_cluster_modes():
|
||||||
|
llm = FakeLLM()
|
||||||
|
settings = _settings()
|
||||||
|
kb = KnowledgeBase("")
|
||||||
|
snapshot = SnapshotProvider(settings)
|
||||||
|
engine = AnswerEngine(settings, llm, kb, snapshot)
|
||||||
|
|
||||||
|
result = asyncio.run(engine.answer("what is 2+2?", mode="quick"))
|
||||||
|
assert "focus on Titan cluster operations" in result.reply
|
||||||
|
|
||||||
|
|
||||||
def test_quick_mode_time_budget_guard():
|
def test_quick_mode_time_budget_guard():
|
||||||
llm = SlowFakeLLM()
|
llm = SlowFakeLLM()
|
||||||
settings = replace(_settings(), quick_time_budget_sec=0.01)
|
settings = replace(_settings(), quick_time_budget_sec=0.01)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user