"""Model upgrades preserve exact identity, capacity, and CLI isolation.""" import json from pathlib import Path import subprocess import sys import pytest SCRIPTS = Path(__file__).resolve().parents[2] / "services/hermes/scripts" sys.path.insert(0, str(SCRIPTS)) import suite_backends from suite_cli_diagnostics import usage_counts from suite_contract import (CLAUDE_MODELS, MODELS, Problem, preflight, prompt, reasoning_selection, validate_request) @pytest.mark.parametrize("model", CLAUDE_MODELS) def test_only_configured_model_is_allowed_and_plugins_are_disabled(model): """Each invocation pins one model, with fallback and customizations disabled.""" command = suite_backends.claude_command(model, 30) settings = json.loads(command[command.index("--settings") + 1]) assert command[command.index("--model") + 1] == model + "[1m]" assert settings["availableModels"] == [model] assert settings["switchModelsOnFlag"] is False assert settings["disableAllHooks"] and settings["disableClaudeAiConnectors"] assert settings["syncClaudeAiPlugins"] is False assert settings["enabledPlugins"]["cc-plugin-agents-md@builtin"] is False assert "--fallback-model" not in command env = suite_backends.claude_environment("/jobs/fresh", "synthetic") assert env["CLAUDE_CODE_MAX_OUTPUT_TOKENS"] == "64000" assert env["CLAUDE_AGENT_SDK_DISABLE_BUILTIN_AGENTS"] == "1" @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-sonnet-5-5"]) @pytest.mark.parametrize("failure", [None, "substitution", "capacity", "plugin"]) def test_new_runtime_envelope_remains_fail_closed(model, failure): """A requested name never substitutes for the actual runtime model identity.""" init = {"type": "system", "subtype": "init", "model": model + "[1m]", "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []} limits = {"contextWindow": 1000000, "maxOutputTokens": 128000, "canonicalModel": model, "provider": "firstParty"} result = {"type": "result", "subtype": "success", "is_error": False, "modelUsage": {model + "[1m]": limits}, "usage": {}, "structured_output": {"groups": []}} if failure == "substitution": limits["canonicalModel"] = "claude-opus-4-8" elif failure == "capacity": limits["contextWindow"] = 200000 elif failure == "plugin": init["plugins"] = [{"name": "unapproved"}] raw = "\n".join(map(json.dumps, (init, result))) if failure: with pytest.raises(Problem): suite_backends.parse_claude(raw, model, exit_code=0) else: _, metadata = suite_backends.parse_claude(raw, model, exit_code=0) assert metadata["model"] == model assert metadata["model_usage"][model + "[1m]"]["maxOutputTokens"] == 128000 def test_unknown_model_cannot_start_a_cli_process(): """Operator configuration and command construction both reject unknown IDs.""" with pytest.raises(Problem): suite_backends.claude_command("unapproved-model", 30) result = subprocess.run([sys.executable, "-c", "import suite_contract"], cwd=SCRIPTS, env={"PLANNING_CLAUDE_MODEL": "unapproved-model"}, capture_output=True, text=True) assert result.returncode != 0 assert "Unsupported configured Claude model" in result.stderr def test_thinking_usage_is_counted_without_retaining_unrecognized_fields(): """New CLI usage details retain measurements and drop arbitrary content.""" assert usage_counts({"output_tokens_details": {"thinking_tokens": 123, "content": "private"}}) == { "output_tokens_details": {"thinking_tokens": 123}} def sized_request(count, source_bytes=None): """Build complete synthetic records with optional exact UTF-8 source size.""" value = {"campaign": "SYNTHETIC", "suite": "EFFORT", "cases": [ {"alias": f"CASE-{i:04d}", "description": "Synthetic parser objective."} for i in range(count)], "routing": {"allow_external": True, "allowed_external_providers": ["claude"]}} if source_bytes is not None: remaining = source_bytes - len(prompt(value).encode("utf-8")) for case in value["cases"]: take = min(remaining, 20000) case["description"] += "\u03bc" * (take // 2) + "x" * (take % 2) remaining -= take assert remaining == 0 return validate_request(value, ["claude"]) @pytest.mark.parametrize("count,effort", [(14, "high"), (75, "high"), (99, "high"), (100, "xhigh"), (363, "xhigh"), (400, "xhigh")]) def test_preflight_selects_effort_at_case_count_boundary(count, effort): """Small and large requests choose independently without changing the default.""" selected = preflight(sized_request(count)) assert selected["provider"] == "claude" and selected["reasoning"] == effort assert selected["reasoning_selection"]["triggers"] == (["case_count"] if count >= 100 else []) command = suite_backends.claude_command(selected["model"], 30, reasoning=selected["reasoning"]) assert command[command.index("--effort") + 1] == effort assert MODELS["claude"]["reasoning"] == "high" @pytest.mark.parametrize("size,effort", [(131071, "high"), (131072, "xhigh"), (131073, "xhigh")]) def test_complete_utf8_source_size_is_an_independent_trigger(size, effort): """Use encoded bytes, not characters or transport whitespace, at the boundary.""" value = sized_request(14, size) selected = preflight(value) assert selected["reasoning"] == effort assert selected["reasoning_selection"]["source_bytes"] == size assert len(prompt(value)) < size assert selected["reasoning_selection"]["triggers"] == (["source_bytes"] if size >= 131072 else []) reordered = {**value, "cases": list(reversed(value["cases"]))} assert reasoning_selection(reordered, "claude") == reasoning_selection(value, "claude") value["execution"]["max_cost_usd"] = 1 assert reasoning_selection(value, "claude")["reasoning"] == effort def test_size_policy_cannot_broaden_routing_or_enable_client_effort_override(): """Effort selection never adds an external destination to a local-only job.""" value = sized_request(363) value["routing"] = {"allow_external": False, "allowed_external_providers": []} with pytest.raises(Problem, match="capacity_or_unsupported_backend"): preflight(value) assert reasoning_selection(value, "local") == {"reasoning": "none"} value["execution"]["reasoning"] = "xhigh" with pytest.raises(Problem, match="invalid_request"): validate_request(value, ["claude"]) with pytest.raises(Problem, match="unsupported_reasoning"): suite_backends.claude_command(MODELS["claude"]["model"], 30, reasoning="max")