atlasbot: guard unsupported claims

atlasbot/engine/answerer.py

@@ -367,6 +367,8 @@ class AnswerEngine:
                                 plan,
                                 max_lines=max(2, plan.max_subquestions),
                             )
+                        if not metric_facts and fallback_candidates:
+                            metric_facts = fallback_candidates[: max(2, plan.max_subquestions)]
                         if metric_facts:
                             key_facts = _merge_fact_lines(metric_facts, key_facts)
                 if self._settings.debug_pipeline:
@@ -532,6 +534,26 @@ class AnswerEngine:
                             )
                 reply = _strip_unknown_entities(reply, unknown_nodes, unknown_namespaces)
 
+            if facts_used and _needs_evidence_guard(reply, facts_used):
+                if observer:
+                    observer("evidence_guard", "tightening unsupported claims")
+                guard_prompt = (
+                    prompts.EVIDENCE_GUARD_PROMPT
+                    + "\nQuestion: "
+                    + normalized
+                    + "\nDraft: "
+                    + reply
+                    + "\nFactsUsed:\n"
+                    + "\n".join(facts_used)
+                )
+                reply = await call_llm(
+                    prompts.EVIDENCE_GUARD_SYSTEM,
+                    guard_prompt,
+                    context=context,
+                    model=plan.model,
+                    tag="evidence_guard",
+                )
+
             if _needs_focus_fix(normalized, reply, classify):
                 if observer:
                     observer("focus_fix", "tightening answer")
@@ -1396,6 +1418,23 @@ def _strip_unknown_entities(reply: str, unknown_nodes: list[str], unknown_namesp
     return cleaned or reply
 
 
+def _needs_evidence_guard(reply: str, facts: list[str]) -> bool:
+    if not reply or not facts:
+        return False
+    lower_reply = reply.lower()
+    fact_text = " ".join(facts).lower()
+    node_pattern = re.compile(r"\b(titan-[0-9a-z]+|node-?\d+)\b", re.IGNORECASE)
+    nodes = {m.group(1).lower() for m in node_pattern.finditer(reply)}
+    if nodes:
+        missing = [node for node in nodes if node not in fact_text]
+        if missing:
+            return True
+    pressure_terms = ("pressure", "diskpressure", "memorypressure", "pidpressure", "headroom")
+    if any(term in lower_reply for term in pressure_terms) and not any(term in fact_text for term in pressure_terms):
+        return True
+    return False
+
+
 def _filter_lines_by_keywords(lines: list[str], keywords: list[str], max_lines: int) -> list[str]:
     if not lines:
         return []

atlasbot/llm/prompts.py

@@ -126,6 +126,19 @@ EVIDENCE_FIX_PROMPT = (
     "If AllowedNamespaces are provided, remove or correct any namespaces not in the list. "
 )
 
+EVIDENCE_GUARD_SYSTEM = (
+    CLUSTER_SYSTEM
+    + " Remove unsupported claims and ensure every node-specific or pressure-related statement is backed by FactsUsed. "
+    + "If FactsUsed is insufficient, answer briefly and say the data is not present."
+)
+
+EVIDENCE_GUARD_PROMPT = (
+    "Rewrite the draft to only include claims supported by FactsUsed. "
+    "If the draft mentions pressure/overload/headroom without evidence, remove it. "
+    "If the draft mentions nodes not in FactsUsed, remove those statements. "
+    "Return the corrected answer only."
+)
+
 RUNBOOK_ENFORCE_SYSTEM = (
     CLUSTER_SYSTEM
     + " Ensure the answer includes the required runbook path. "