diff --git a/scripts/ops/hermes_batch_client.py b/scripts/ops/hermes_batch_client.py index 386bf450..cc2d5ad7 100755 --- a/scripts/ops/hermes_batch_client.py +++ b/scripts/ops/hermes_batch_client.py @@ -91,7 +91,7 @@ class BatchClient: raise ValueError("approved runtime/model configuration mismatch") return result - def run(self, envelope): + def run(self, envelope, validator=None): """Cache a FA01/synthetic request by source, prompt, schema and model identity. The roster application must filter source rows and permitted fields before @@ -118,6 +118,7 @@ class BatchClient: self._verify(record["result"], payload) if record.get("cache_key") != key: raise ValueError("cache identity mismatch") + self._validate_output(destination, record, validator) return {"cache_hit": True, "path": str(destination), "record": record} self.catalog() started = time.monotonic() @@ -128,9 +129,21 @@ class BatchClient: "request_sha256": hashlib.sha256(canonical(payload)).hexdigest(), "campaign_id": envelope["campaign_id"], "suite_id": envelope["suite_id"], "client_wall_seconds": round(time.monotonic() - started, 3), "result": result} + self._validate_output(destination, record, validator) self._save(destination, record) return {"cache_hit": False, "path": str(destination), "record": record} + def _validate_output(self, destination, record, validator): + """Keep failed application contracts as diagnostics, outside reusable cache.""" + if validator is None: + return + try: + validator(json.loads(record["result"]["response"])) + except (ValueError, KeyError, TypeError): + self._save(destination.with_suffix(".rejected.json"), record) + destination.unlink(missing_ok=True) + raise ValueError("application validation failed; local diagnostic retained") from None + @staticmethod def _verify(result, payload): """Reject changed models, settings and incomplete results before caching.""" diff --git a/scripts/ops/test_planning/synthetic_probe.py b/scripts/ops/test_planning/synthetic_probe.py index 81cbea24..5b540a43 100755 --- a/scripts/ops/test_planning/synthetic_probe.py +++ b/scripts/ops/test_planning/synthetic_probe.py @@ -64,16 +64,17 @@ def main(): parser.error("case-id must select one bundled synthetic case") request = envelope(args.model, args.mode, sources, args.think) client = BatchClient(args.token_file, args.cache_dir) - outcome = client.run(request) + def validate(content): + if args.mode == "profile": + validate_profile(content, sources[0]) + elif args.mode == "group": + validate_families(content, sources) + elif content != {"status": "LOCAL_OK"}: + raise ValueError("synthetic transport check failed") + + outcome = client.run(request, validator=validate) record = outcome["record"] result = record["result"] - content = json.loads(result["response"]) - if args.mode == "profile": - validate_profile(content, sources[0]) - elif args.mode == "group": - validate_families(content, sources) - elif content != {"status": "LOCAL_OK"}: - raise ValueError("synthetic transport check failed") print(json.dumps({"synthetic": True, "mode": args.mode, "model": args.model, "cache_hit": outcome["cache_hit"], "result_file": outcome["path"], "wall_seconds": record["client_wall_seconds"], diff --git a/testing/tests/test_hermes_batch_client.py b/testing/tests/test_hermes_batch_client.py index 3dc33e4d..7df66123 100644 --- a/testing/tests/test_hermes_batch_client.py +++ b/testing/tests/test_hermes_batch_client.py @@ -128,3 +128,17 @@ def test_redirects_and_shared_token_files_are_rejected(tmp_path): token.chmod(0o644) with pytest.raises(ValueError): client.BatchClient(token, tmp_path / "cache") + + +def test_application_validation_failure_keeps_diagnostics_but_is_not_reused(transport, envelope): + instance, calls = transport + + def reject(content): + raise ValueError("missing case IDs") + + for _ in range(2): + with pytest.raises(ValueError, match="application validation failed"): + instance.run(envelope, validator=reject) + assert calls.count("/generate") == 2 + assert len(list(instance.cache_dir.glob("*.rejected.json"))) == 1 + assert len(list(instance.cache_dir.glob("*.json"))) == 1