102 lines
2.8 KiB
Python
102 lines
2.8 KiB
Python
"""Claude subscription broker tool-boundary regression tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "services/hermes/scripts"
|
|
|
|
|
|
def _module(monkeypatch):
|
|
routing = ModuleType("routing_catalog")
|
|
routing.load_catalog = lambda: {}
|
|
routing.resolve_route = lambda route: route
|
|
monkeypatch.setitem(sys.modules, "routing_catalog", routing)
|
|
spec = importlib.util.spec_from_file_location(
|
|
"claude_broker_tool_test", SCRIPTS / "claude_oauth_broker.py"
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _payload(user: str = "Implement and test the fix") -> dict:
|
|
return {
|
|
"messages": [{"role": "user", "content": user}],
|
|
"tools": [
|
|
{"name": "terminal", "description": "Run a command"},
|
|
{"name": "todo", "description": "Track work"},
|
|
],
|
|
}
|
|
|
|
|
|
def test_prompt_declares_external_tools_available(monkeypatch) -> None:
|
|
module = _module(monkeypatch)
|
|
|
|
prompt = module._prompt(_payload())
|
|
|
|
assert "available external Hermes tool" in prompt
|
|
assert "Never simulate a tool result" in prompt
|
|
|
|
|
|
def test_valid_external_tool_request_is_accepted(monkeypatch) -> None:
|
|
module = _module(monkeypatch)
|
|
|
|
module._validate_structured(
|
|
{
|
|
"type": "tool_calls",
|
|
"text": "",
|
|
"tool_calls": [{"name": "terminal", "input": {"command": "git status"}}],
|
|
},
|
|
_payload(),
|
|
)
|
|
|
|
|
|
def test_fabricated_tool_unavailability_is_retryable(monkeypatch) -> None:
|
|
module = _module(monkeypatch)
|
|
|
|
with pytest.raises(RuntimeError, match="fabricated"):
|
|
module._validate_structured(
|
|
{
|
|
"type": "final",
|
|
"text": "The terminal returned No such tool available.",
|
|
"tool_calls": [],
|
|
},
|
|
_payload(),
|
|
)
|
|
|
|
|
|
def test_user_can_ask_about_a_prior_tool_error(monkeypatch) -> None:
|
|
module = _module(monkeypatch)
|
|
|
|
module._validate_structured(
|
|
{
|
|
"type": "final",
|
|
"text": "No such tool available means terminal was not exposed then.",
|
|
"tool_calls": [],
|
|
},
|
|
_payload("Why did terminal report No such tool available?"),
|
|
)
|
|
|
|
|
|
def test_unadvertised_tool_call_is_retryable(monkeypatch) -> None:
|
|
module = _module(monkeypatch)
|
|
|
|
with pytest.raises(RuntimeError, match="unavailable tool call"):
|
|
module._validate_structured(
|
|
{
|
|
"type": "tool_calls",
|
|
"text": "",
|
|
"tool_calls": [{"name": "shell", "input": {}}],
|
|
},
|
|
_payload(),
|
|
)
|