diff --git a/atlasbot/engine/answerer.py b/atlasbot/engine/answerer.py index b30862f..72e5944 100644 --- a/atlasbot/engine/answerer.py +++ b/atlasbot/engine/answerer.py @@ -367,8 +367,10 @@ class AnswerEngine: plan, max_lines=max(2, plan.max_subquestions), ) - if metric_facts: - key_facts = _merge_fact_lines(metric_facts, key_facts) + if not metric_facts and fallback_candidates: + metric_facts = fallback_candidates[: max(2, plan.max_subquestions)] + if metric_facts: + key_facts = _merge_fact_lines(metric_facts, key_facts) if self._settings.debug_pipeline: scored_preview = sorted( [{"id": c["id"], "score": scored.get(c["id"], 0.0), "summary": c["summary"]} for c in chunks], @@ -532,6 +534,26 @@ class AnswerEngine: ) reply = _strip_unknown_entities(reply, unknown_nodes, unknown_namespaces) + if facts_used and _needs_evidence_guard(reply, facts_used): + if observer: + observer("evidence_guard", "tightening unsupported claims") + guard_prompt = ( + prompts.EVIDENCE_GUARD_PROMPT + + "\nQuestion: " + + normalized + + "\nDraft: " + + reply + + "\nFactsUsed:\n" + + "\n".join(facts_used) + ) + reply = await call_llm( + prompts.EVIDENCE_GUARD_SYSTEM, + guard_prompt, + context=context, + model=plan.model, + tag="evidence_guard", + ) + if _needs_focus_fix(normalized, reply, classify): if observer: observer("focus_fix", "tightening answer") @@ -1396,6 +1418,23 @@ def _strip_unknown_entities(reply: str, unknown_nodes: list[str], unknown_namesp return cleaned or reply +def _needs_evidence_guard(reply: str, facts: list[str]) -> bool: + if not reply or not facts: + return False + lower_reply = reply.lower() + fact_text = " ".join(facts).lower() + node_pattern = re.compile(r"\b(titan-[0-9a-z]+|node-?\d+)\b", re.IGNORECASE) + nodes = {m.group(1).lower() for m in node_pattern.finditer(reply)} + if nodes: + missing = [node for node in nodes if node not in fact_text] + if missing: + return True + pressure_terms = ("pressure", "diskpressure", "memorypressure", "pidpressure", "headroom") + if any(term in lower_reply for term in pressure_terms) and not any(term in fact_text for term in pressure_terms): + return True + return False + + def _filter_lines_by_keywords(lines: list[str], keywords: list[str], max_lines: int) -> list[str]: if not lines: return [] diff --git a/atlasbot/llm/prompts.py b/atlasbot/llm/prompts.py index 383f21a..50d26d0 100644 --- a/atlasbot/llm/prompts.py +++ b/atlasbot/llm/prompts.py @@ -126,6 +126,19 @@ EVIDENCE_FIX_PROMPT = ( "If AllowedNamespaces are provided, remove or correct any namespaces not in the list. " ) +EVIDENCE_GUARD_SYSTEM = ( + CLUSTER_SYSTEM + + " Remove unsupported claims and ensure every node-specific or pressure-related statement is backed by FactsUsed. " + + "If FactsUsed is insufficient, answer briefly and say the data is not present." +) + +EVIDENCE_GUARD_PROMPT = ( + "Rewrite the draft to only include claims supported by FactsUsed. " + "If the draft mentions pressure/overload/headroom without evidence, remove it. " + "If the draft mentions nodes not in FactsUsed, remove those statements. " + "Return the corrected answer only." +) + RUNBOOK_ENFORCE_SYSTEM = ( CLUSTER_SYSTEM + " Ensure the answer includes the required runbook path. "