"""Model upgrades preserve exact identity, capacity, and CLI isolation.""" import json from pathlib import Path import subprocess import sys import pytest SCRIPTS = Path(__file__).resolve().parents[2] / "services/hermes/scripts" sys.path.insert(0, str(SCRIPTS)) import suite_backends from suite_cli_diagnostics import usage_counts from suite_contract import CLAUDE_MODELS, Problem @pytest.mark.parametrize("model", CLAUDE_MODELS) def test_only_configured_model_is_allowed_and_plugins_are_disabled(model): """Each invocation pins one model, with fallback and customizations disabled.""" command = suite_backends.claude_command(model, 30) settings = json.loads(command[command.index("--settings") + 1]) assert command[command.index("--model") + 1] == model + "[1m]" assert settings["availableModels"] == [model] assert settings["switchModelsOnFlag"] is False assert settings["disableAllHooks"] and settings["disableClaudeAiConnectors"] assert settings["syncClaudeAiPlugins"] is False assert settings["enabledPlugins"]["cc-plugin-agents-md@builtin"] is False assert "--fallback-model" not in command env = suite_backends.claude_environment("/jobs/fresh", "synthetic") assert env["CLAUDE_CODE_MAX_OUTPUT_TOKENS"] == "64000" assert env["CLAUDE_AGENT_SDK_DISABLE_BUILTIN_AGENTS"] == "1" @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-sonnet-5-5"]) @pytest.mark.parametrize("failure", [None, "substitution", "capacity", "plugin"]) def test_new_runtime_envelope_remains_fail_closed(model, failure): """A requested name never substitutes for the actual runtime model identity.""" init = {"type": "system", "subtype": "init", "model": model + "[1m]", "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []} limits = {"contextWindow": 1000000, "maxOutputTokens": 128000, "canonicalModel": model, "provider": "firstParty"} result = {"type": "result", "subtype": "success", "is_error": False, "modelUsage": {model + "[1m]": limits}, "usage": {}, "structured_output": {"groups": []}} if failure == "substitution": limits["canonicalModel"] = "claude-opus-4-8" elif failure == "capacity": limits["contextWindow"] = 200000 elif failure == "plugin": init["plugins"] = [{"name": "unapproved"}] raw = "\n".join(map(json.dumps, (init, result))) if failure: with pytest.raises(Problem): suite_backends.parse_claude(raw, model, exit_code=0) else: _, metadata = suite_backends.parse_claude(raw, model, exit_code=0) assert metadata["model"] == model assert metadata["model_usage"][model + "[1m]"]["maxOutputTokens"] == 128000 def test_unknown_model_cannot_start_a_cli_process(): """Operator configuration and command construction both reject unknown IDs.""" with pytest.raises(Problem): suite_backends.claude_command("unapproved-model", 30) result = subprocess.run([sys.executable, "-c", "import suite_contract"], cwd=SCRIPTS, env={"PLANNING_CLAUDE_MODEL": "unapproved-model"}, capture_output=True, text=True) assert result.returncode != 0 assert "Unsupported configured Claude model" in result.stderr def test_thinking_usage_is_counted_without_retaining_unrecognized_fields(): """New CLI usage details retain measurements and drop arbitrary content.""" assert usage_counts({"output_tokens_details": {"thinking_tokens": 123, "content": "private"}}) == { "output_tokens_details": {"thinking_tokens": 123}}