triage: constrain local diagnosis schema

ariadne/services/testing_triage_diagnosis.py

@@ -15,11 +15,45 @@ _MAX_MODEL_EVIDENCE_CHARS = 24000
 _MAX_MODEL_OUTPUT_CHARS = 12000
 _DIAGNOSIS_SYSTEM_PROMPT = (
     "You are Ariadne's local testing triage model. Use only the supplied JSON evidence. "
-    "Return JSON only with keys: headline, root_cause, blast_radius, confidence, "
-    "needs_human, next_actions, evidence_refs. Confidence must be low, medium, or high. "
-    "Next actions must be read-only verification or Flux/IaC changes; never suggest "
-    "mutating kubectl commands or reading Kubernetes Secret values."
+    "Return exactly one diagnosis object that matches the requested schema. Do not copy "
+    "or summarize the evidence as nested input data. Confidence must be low, medium, or "
+    "high. Next actions must be read-only verification or Flux/IaC changes; never suggest "
+    "mutating kubectl commands or reading Kubernetes Secret values. If evidence is "
+    "insufficient, say that plainly in root_cause. Use normal English words with spaces; "
+    "do not concatenate words."
 )
+_DIAGNOSIS_RESPONSE_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "headline": {"type": "string"},
+        "root_cause": {"type": "string"},
+        "blast_radius": {"type": "string"},
+        "confidence": {"type": "string", "enum": ["low", "medium", "high"]},
+        "needs_human": {"type": "boolean"},
+        "next_actions": {
+            "type": "array",
+            "items": {"type": "string"},
+            "minItems": 1,
+            "maxItems": 6,
+        },
+        "evidence_refs": {
+            "type": "array",
+            "items": {"type": "string"},
+            "minItems": 1,
+            "maxItems": 8,
+        },
+    },
+    "required": [
+        "headline",
+        "root_cause",
+        "blast_radius",
+        "confidence",
+        "needs_human",
+        "next_actions",
+        "evidence_refs",
+    ],
+}
 
 
 def latest_testing_triage_diagnosis(storage: Storage) -> dict[str, Any] | None:
@@ -62,7 +96,7 @@ def diagnose_testing_triage(bundle: dict[str, Any]) -> dict[str, Any]:
                     "system": _DIAGNOSIS_SYSTEM_PROMPT,
                     "prompt": _diagnosis_prompt(bundle),
                     "stream": False,
-                    "format": "json",
+                    "format": _DIAGNOSIS_RESPONSE_SCHEMA,
                     "options": {"temperature": 0.1, "top_p": 0.9},
                 },
             )
@@ -90,16 +124,15 @@ def _model_timeout() -> float:
 
 def _diagnosis_prompt(bundle: dict[str, Any]) -> str:
     payload = {
-        "task": "Summarize testing and cluster evidence for tonight's debugging work.",
-        "required_output_schema": {
-            "headline": "one sentence",
-            "root_cause": "most likely cause, or say evidence is insufficient",
-            "blast_radius": "affected suites, namespaces, pods, nodes, or services",
-            "confidence": "low|medium|high",
-            "needs_human": True,
-            "next_actions": ["short, concrete actions"],
-            "evidence_refs": ["specific evidence keys or values used"],
-        },
+        "task": (
+            "Diagnose the highest-priority testing or cluster issue for tonight. "
+            "Write a compact triage finding, not an evidence dump."
+        ),
+        "output_rules": [
+            "Return only headline, root_cause, blast_radius, confidence, needs_human, next_actions, evidence_refs.",
+            "Do not include keys named pipelines, quality, bundle, evidence, summary, or unknowns.",
+            "Prefer concrete Jenkins job names, Flux Kustomizations, pod names, nodes, and metrics when present.",
+        ],
         "bundle": _model_evidence_payload(bundle),
     }
     evidence = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)

tests/test_testing_triage_diagnosis.py

@@ -111,7 +111,9 @@ def test_diagnose_testing_triage_calls_local_ollama(monkeypatch) -> None:
     assert captured["url"] == "http://ollama/api/generate"
     assert captured["timeout"] == 3.0
     assert captured["request"]["model"] == "tiny-model"
-    assert captured["request"]["format"] == "json"
+    assert captured["request"]["format"]["additionalProperties"] is False
+    assert "headline" in captured["request"]["format"]["required"]
+    assert "Do not include keys named pipelines" in captured["request"]["prompt"]
     assert diagnosis["kind"] == "testing_triage_diagnosis"
     assert diagnosis["status"] == "needs_attention"
     assert diagnosis["diagnosis"]["confidence"] == "medium"