"""Critical policy, completeness, isolation, and retry guarantees for suite jobs.""" import copy import json from pathlib import Path import sys import threading import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "services/hermes/scripts")) import suite_api import suite_backends from suite_contract import (CLAUDE_MODELS, MODELS, PROMPT_REVISION, PROMPT_SHA256, SYSTEM, Problem, preflight, prompt, validate_partition, validate_request, validate_result) from suite_jobs import Jobs from suite_synthetic import fixture, score def external(size=14): request = fixture(size)[0] request["routing"] = {"allow_external": True, "allowed_external_providers": ["claude"]} return suite_api.authorize(request, ["claude"]) def expected_result(size): _, expected = fixture(size) groups = {} for alias, family in expected.items(): groups.setdefault(family, []).append(alias) return {"groups": [{"name": name, "description": "Shared synthetic implementation machinery", "members": members} for name, members in groups.items()]} @pytest.mark.parametrize("size", [14, 75, 363]) def test_full_suite_capacity_and_coverage(size): request = external(size) selection = preflight(request) assert selection["provider"] == "claude" assert selection["case_count"] == size # Every field and record, including duplicate text, survives preparation exactly. assert json.loads(prompt(request))["cases"] == request["cases"] assert len({case["alias"] for case in request["cases"]}) == size result = validate_partition(expected_result(size), request) assert score(result, fixture(size)[1])["pair_recall"] == 1 result["groups"][0]["members"].append("[reference]") with pytest.raises(Problem, match="invalid_case_assignments"): validate_partition(result, request) @pytest.mark.parametrize("policy", [ {"allow_external": "false"}, {"allow_external": 1}, {"allow_external": False, "allowed_external_providers": ["claude"]}, {"allow_external": True, "allowed_external_providers": []}, {"allow_external": True, "allowed_external_providers": ["unknown"]}, {"allow_external": True, "allowed_external_providers": ["claude", "claude"]}, {"allow_external": True, "allowed_external_providers": "claude"}, ]) def test_malformed_policy_rejected(policy): request = fixture(14)[0] request["routing"] = policy with pytest.raises(Problem): suite_api.authorize(request, ["claude"]) def test_permissions_and_external_data_scope(monkeypatch): monkeypatch.delenv("PLANNING_GENERALIZED_CLAUDE_APPROVED", raising=False) with pytest.raises(Problem, match="provider_forbidden"): suite_api.authorize(external(), []) request = external() request["cases"][0]["description"] = "Changed input requires separate approval" with pytest.raises(Problem, match="external_data_not_approved"): suite_api.authorize(request, ["claude"]) def test_generalized_approval_remains_claude_and_credential_scoped(monkeypatch): monkeypatch.setenv("PLANNING_GENERALIZED_CLAUDE_APPROVED", "true") request = external() request["cases"][0]["description"] = "Synthetic example of a generalized input" with pytest.raises(Problem, match="external_data_not_approved"): suite_api.authorize(request, ["claude"]) assert suite_api.authorize(request, ["claude"], operational=True)["cases"] == request["cases"] with pytest.raises(Problem, match="provider_forbidden"): suite_api.authorize(request, [], operational=True) for providers in (["codex"], ["claude", "codex"]): request["routing"]["allowed_external_providers"] = providers with pytest.raises(Problem, match="provider_forbidden"): suite_api.authorize(request, providers, operational=True) def test_local_schema_excludes_descriptions_from_members(monkeypatch): from types import SimpleNamespace captured = [] request = validate_request({"campaign": "SYNTHETIC", "suite": "PARSER", "cases": [ {"alias": "CASE-1", "description": "Parse valid configuration text"}]}, []) def respond(url, value, *args): captured.append(value) return {"done": True, "model": MODELS["local"]["model"], "response": '{"groups":[]}'} monkeypatch.setattr(suite_backends, "post", respond) monkeypatch.setattr(suite_backends, "Path", lambda _: SimpleNamespace(read_text=lambda: "fake")) suite_backends.local_generate(request, threading.Event(), "192.168.22.8") assert captured[0]["prompt"] == SYSTEM + "\n" + prompt(request) items = captured[0]["format"]["properties"]["groups"]["items"]["properties"]["members"]["items"] assert items["enum"] == ["CASE-1"] assert "enum" not in suite_backends.SCHEMA["properties"]["groups"]["items"]["properties"]["members"]["items"] def test_local_default_cannot_overflow_to_provider(): request = validate_request(fixture(14)[0], ["claude", "codex"]) assert request["routing"] == {"allow_external": False, "allowed_external_providers": []} with pytest.raises(Problem, match="capacity_or_unsupported_backend") as error: preflight(request) assert set(error.value.details["candidates"]) == {"local"} def test_codex_unverified_capacity_fails_closed(): request = fixture(14)[0] request["routing"] = {"allow_external": True, "allowed_external_providers": ["codex"]} with pytest.raises(Problem, match="capacity_or_unsupported_backend"): preflight(suite_api.authorize(request, ["codex"])) @pytest.mark.parametrize("mutation", ["duplicate", "ownership", "alias", "unknown", "strategy"]) def test_bad_membership_and_contract(mutation): request = fixture(14)[0] if mutation == "duplicate": request["cases"][1]["alias"] = request["cases"][0]["alias"] elif mutation == "ownership": request["cases"][1]["suite"] = "different" elif mutation == "alias": request["cases"][0]["alias"] = "[reference]" elif mutation == "unknown": request["cases"][0]["raw"] = {"secret": "not permitted"} else: request["execution"] = {"strategy": "independent_batches"} with pytest.raises(Problem): validate_request(request, []) def test_output_assignment_errors(): request = external() for mutation in ("omitted", "duplicate", "invented"): result = expected_result(14) members = result["groups"][0]["members"] if mutation == "omitted": members.pop() elif mutation == "duplicate": members.append(members[0]) else: members[0] = "CASE-INVENTED" with pytest.raises(Problem): validate_result(result, request) def test_idempotency_ownership_busy_restart(tmp_path): jobs = Jobs(tmp_path / "jobs.sqlite") request = external() selection = preflight(request) first, created = jobs.submit("owner", "same-key", request, selection, "192.168.22.8", launch=False) assert created repeat, created = jobs.submit("owner", "same-key", request, selection, "192.168.22.8", launch=False) assert not created and repeat["job_id"] == first["job_id"] with pytest.raises(Problem, match="idempotency_conflict"): jobs.submit("owner", "same-key", external(75), selection, "192.168.22.8", launch=False) with pytest.raises(Problem, match="capacity_busy"): jobs.submit("owner", "another-key", request, selection, "192.168.22.8", launch=False) with pytest.raises(Problem, match="job_not_found"): jobs.get(first["job_id"], "other-owner") restarted = Jobs(tmp_path / "jobs.sqlite") assert restarted.get(first["job_id"], "owner")["error"]["code"] == "interrupted_no_retry" assert not restarted.submit("owner", "same-key", request, selection, "192.168.22.8", launch=False)[1] assert b"thermal chamber" not in (tmp_path / "jobs.sqlite").read_bytes() def test_no_fallback_after_local_failure(tmp_path, monkeypatch, capsys): request = {"campaign": "SYNTHETIC", "suite": "TINY", "cases": [ {"alias": "CASE-1", "description": "Read parser status"}]} request = validate_request(request, []) import suite_multipass # Isolate runtime fail-closed behavior from the new workflow admission limit. selected = {"provider": "local", **MODELS["local"]} monkeypatch.setattr(suite_multipass, "capacity", lambda *a: {}) calls = [] monkeypatch.setattr(suite_backends, "switchyard_decision", lambda provider: calls.append(provider)) def fail(*args, **kwargs): raise Problem("backend_unavailable", 503) monkeypatch.setattr(suite_backends, "local_generate", fail) monkeypatch.setattr(suite_backends, "claude_generate", lambda *args, **kwargs: pytest.fail("external launch")) jobs = Jobs(tmp_path / "jobs.sqlite") document, _ = jobs.submit("owner", "local-failure", request, selected, "192.168.22.8", launch=False) jobs.run(document["job_id"], "owner", request, selected, "192.168.22.8") assert calls == ["local"] assert jobs.get(document["job_id"], "owner")["status"] == "failed" assert "Read parser status" not in capsys.readouterr().out def test_fresh_cli_isolation_and_schema(): command = suite_backends.claude_command(MODELS["claude"]["model"], 5) assert "--safe-mode" in command and "--no-session-persistence" in command assert not any("bypass" in flag or "resume" in flag for flag in command) assert command[command.index("--tools") + 1] == "" assert command[command.index("--system-prompt") + 1] == SYSTEM assert json.loads(command[command.index("--json-schema") + 1]) == suite_backends.SCHEMA env = suite_backends.claude_environment("/jobs/fresh", "fake-token") assert env["DISABLE_COMPACT"] == "1" assert env["CLAUDE_CODE_MAX_RETRIES"] == "0" assert "ANTHROPIC_API_KEY" not in env def test_prompt_provenance_is_stored_and_replay_does_not_relabel(tmp_path, monkeypatch): import hashlib import suite_jobs request = external() selection = preflight(request) assert selection["prompt_revision"] == PROMPT_REVISION assert selection["prompt_sha256"] == hashlib.sha256(SYSTEM.encode()).hexdigest() jobs = Jobs(tmp_path / "jobs.sqlite") document, _ = jobs.submit("owner", "prompt-version-key", request, selection, "192.168.22.8", launch=False) monkeypatch.setattr(suite_jobs, "PROMPT_REVISION", "future-prompt") monkeypatch.setattr(suite_jobs, "PROMPT_SHA256", "future-hash") replay, created = jobs.submit("owner", "prompt-version-key", request, selection, "192.168.22.8", launch=False) assert not created and replay["job_id"] == document["job_id"] assert replay["prompt_revision"] == PROMPT_REVISION assert replay["prompt_sha256"] == PROMPT_SHA256 restarted = Jobs(tmp_path / "jobs.sqlite") assert restarted.get(document["job_id"], "owner")["prompt_revision"] == PROMPT_REVISION def test_compaction_and_incomplete_detection(): for events, code in [([{"type": "system", "subtype": "compact_boundary"}], "compaction_detected"), ([], "pass_missing_terminal_event")]: with pytest.raises(Problem, match=code): suite_backends.parse_claude("\n".join(json.dumps(e) for e in events), "claude-fable-5") @pytest.mark.parametrize("failure,code", [ (None, None), ("model", "model_changed"), ("context", "backend_capabilities_changed"), ("tools", "worker_isolation_failed"), ("output", "pass_output_limit"), ]) def test_actual_cli_envelope_and_runtime_guards(failure, code): model = MODELS["claude"]["model"] init = {"type": "system", "subtype": "init", "model": model + "[1m]", "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []} limits = {"contextWindow": 1000000, "maxOutputTokens": CLAUDE_MODELS[model], "canonicalModel": model, "provider": "firstParty"} result = {"type": "result", "subtype": "success", "is_error": False, "modelUsage": {model + "[1m]": limits}, "usage": {}, "structured_output": expected_result(14)} if failure == "model": result["modelUsage"] = {"claude-unexpected": limits} elif failure == "context": limits["contextWindow"] = 200000 elif failure == "tools": init["tools"].append("Bash") elif failure == "output": result["stop_reason"] = "max_tokens" raw = "\n".join(json.dumps(e) for e in [init, result]) if code: with pytest.raises(Problem, match=code): suite_backends.parse_claude(raw, model) else: value, metadata = suite_backends.parse_claude(raw, model) assert value == expected_result(14) and metadata["model"] == model @pytest.mark.parametrize("raw", ['{"routing":{},"routing":{}}', '{"x":NaN}', '{"x":Infinity}', '{']) def test_strict_json(raw): with pytest.raises(Problem, match="invalid_json"): suite_api.strict_json(raw) def test_credential_permissions(tmp_path): (tmp_path / "token").write_text("local-secret") (tmp_path / "synthetic-token").write_text("synthetic-secret") (tmp_path / "operational-token").write_text("operational-secret") assert suite_api.credential("Bearer local-secret", tmp_path)[1] == [] assert suite_api.credential("Bearer synthetic-secret", tmp_path)[1] == ["claude", "codex"] assert suite_api.credential("Bearer operational-secret", tmp_path)[1] == ["claude"] with pytest.raises(Problem, match="authentication"): suite_api.credential("Bearer invalid", tmp_path)