ai: keep invalid planning results out of reusable cache

scripts/ops/hermes_batch_client.py

@@ -91,7 +91,7 @@ class BatchClient:
             raise ValueError("approved runtime/model configuration mismatch")
         return result
 
-    def run(self, envelope):
+    def run(self, envelope, validator=None):
         """Cache a FA01/synthetic request by source, prompt, schema and model identity.
 
         The roster application must filter source rows and permitted fields before
@@ -118,6 +118,7 @@ class BatchClient:
             self._verify(record["result"], payload)
             if record.get("cache_key") != key:
                 raise ValueError("cache identity mismatch")
+            self._validate_output(destination, record, validator)
             return {"cache_hit": True, "path": str(destination), "record": record}
         self.catalog()
         started = time.monotonic()
@@ -128,9 +129,21 @@ class BatchClient:
                   "request_sha256": hashlib.sha256(canonical(payload)).hexdigest(),
                   "campaign_id": envelope["campaign_id"], "suite_id": envelope["suite_id"],
                   "client_wall_seconds": round(time.monotonic() - started, 3), "result": result}
+        self._validate_output(destination, record, validator)
         self._save(destination, record)
         return {"cache_hit": False, "path": str(destination), "record": record}
 
+    def _validate_output(self, destination, record, validator):
+        """Keep failed application contracts as diagnostics, outside reusable cache."""
+        if validator is None:
+            return
+        try:
+            validator(json.loads(record["result"]["response"]))
+        except (ValueError, KeyError, TypeError):
+            self._save(destination.with_suffix(".rejected.json"), record)
+            destination.unlink(missing_ok=True)
+            raise ValueError("application validation failed; local diagnostic retained") from None
+
     @staticmethod
     def _verify(result, payload):
         """Reject changed models, settings and incomplete results before caching."""

scripts/ops/test_planning/synthetic_probe.py

@@ -64,16 +64,17 @@ def main():
             parser.error("case-id must select one bundled synthetic case")
     request = envelope(args.model, args.mode, sources, args.think)
     client = BatchClient(args.token_file, args.cache_dir)
-    outcome = client.run(request)
+    def validate(content):
+        if args.mode == "profile":
+            validate_profile(content, sources[0])
+        elif args.mode == "group":
+            validate_families(content, sources)
+        elif content != {"status": "LOCAL_OK"}:
+            raise ValueError("synthetic transport check failed")
+
+    outcome = client.run(request, validator=validate)
     record = outcome["record"]
     result = record["result"]
-    content = json.loads(result["response"])
-    if args.mode == "profile":
-        validate_profile(content, sources[0])
-    elif args.mode == "group":
-        validate_families(content, sources)
-    elif content != {"status": "LOCAL_OK"}:
-        raise ValueError("synthetic transport check failed")
     print(json.dumps({"synthetic": True, "mode": args.mode, "model": args.model,
                       "cache_hit": outcome["cache_hit"], "result_file": outcome["path"],
                       "wall_seconds": record["client_wall_seconds"],

testing/tests/test_hermes_batch_client.py

@@ -128,3 +128,17 @@ def test_redirects_and_shared_token_files_are_rejected(tmp_path):
     token.chmod(0o644)
     with pytest.raises(ValueError):
         client.BatchClient(token, tmp_path / "cache")
+
+
+def test_application_validation_failure_keeps_diagnostics_but_is_not_reused(transport, envelope):
+    instance, calls = transport
+
+    def reject(content):
+        raise ValueError("missing case IDs")
+
+    for _ in range(2):
+        with pytest.raises(ValueError, match="application validation failed"):
+            instance.run(envelope, validator=reject)
+    assert calls.count("/generate") == 2
+    assert len(list(instance.cache_dir.glob("*.rejected.json"))) == 1
+    assert len(list(instance.cache_dir.glob("*.json"))) == 1