atlas-iac/testing/tests/test_hermes_claude_broker.py

"""Claude subscription broker tool-boundary regression tests."""

from __future__ import annotations

import importlib.util
import json
import sys
from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
from types import ModuleType, SimpleNamespace

import pytest


ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"


def _module(monkeypatch):
    routing = ModuleType("routing_catalog")
    routing.load_catalog = lambda: {}
    routing.resolve_route = lambda route: route
    monkeypatch.setitem(sys.modules, "routing_catalog", routing)
    spec = importlib.util.spec_from_file_location(
        "claude_broker_tool_test", SCRIPTS / "claude_oauth_broker.py"
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _payload(user: str = "Implement and test the fix") -> dict:
    return {
        "messages": [{"role": "user", "content": user}],
        "tools": [
            {"name": "terminal", "description": "Run a command"},
            {"name": "todo", "description": "Track work"},
        ],
    }


def test_prompt_declares_external_tools_available(monkeypatch) -> None:
    module = _module(monkeypatch)

    prompt = module._prompt(_payload())

    assert "available external Hermes tool" in prompt
    assert "Never simulate a tool result" in prompt


def test_valid_external_tool_request_is_accepted(monkeypatch) -> None:
    module = _module(monkeypatch)

    module._validate_structured(
        {
            "type": "tool_calls",
            "text": "",
            "tool_calls": [{"name": "terminal", "input": {"command": "git status"}}],
        },
        _payload(),
    )


def test_fabricated_tool_unavailability_is_retryable(monkeypatch) -> None:
    module = _module(monkeypatch)

    with pytest.raises(RuntimeError, match="fabricated"):
        module._validate_structured(
            {
                "type": "final",
                "text": "The terminal returned No such tool available.",
                "tool_calls": [],
            },
            _payload(),
        )


def test_user_can_ask_about_a_prior_tool_error(monkeypatch) -> None:
    module = _module(monkeypatch)

    module._validate_structured(
        {
            "type": "final",
            "text": "No such tool available means terminal was not exposed then.",
            "tool_calls": [],
        },
        _payload("Why did terminal report No such tool available?"),
    )


def test_unadvertised_tool_call_is_retryable(monkeypatch) -> None:
    module = _module(monkeypatch)

    with pytest.raises(RuntimeError, match="unavailable tool call"):
        module._validate_structured(
            {
                "type": "tool_calls",
                "text": "",
                "tool_calls": [{"name": "shell", "input": {}}],
            },
            _payload(),
        )


@pytest.mark.parametrize("disconnect", [BrokenPipeError, ConnectionResetError])
def test_json_response_tolerates_disconnected_client(monkeypatch, disconnect) -> None:
    module = _module(monkeypatch)
    handler = object.__new__(module.Handler)
    handler.request_version = "HTTP/1.1"
    handler.command = "GET"
    handler.send_response = lambda _status: None
    handler.send_header = lambda _name, _value: None
    handler.end_headers = lambda: None

    class Disconnected(BytesIO):
        def write(self, _body):
            raise disconnect

    handler.wfile = Disconnected()

    handler._json(200, {"ok": True})


def test_transient_auth_probe_keeps_recent_proven_access(monkeypatch, tmp_path) -> None:
    module = _module(monkeypatch)
    module.HEALTH_PATH = tmp_path / "claude.json"
    module.HEALTH_PATH.write_text(
        json.dumps(
            {
                "authenticated": True,
                "api_provider": "firstParty",
                "auth_method": "oauth_token",
                "last_success_at": datetime.now(timezone.utc).isoformat(),
            }
        ),
        encoding="utf-8",
    )
    monkeypatch.setattr(
        module.subprocess,
        "run",
        lambda *_args, **_kwargs: SimpleNamespace(
            returncode=1, stdout="", stderr="transient timeout"
        ),
    )

    health = module._subscription_health(force=True)

    assert health["authenticated"] is True
    assert health["state"] == "available"
    assert health["api_provider"] == "firstParty"
    assert health["auth_method"] == "oauth_token"
    assert health["auth_probe_stale"] is True


def test_transient_auth_probe_keeps_older_proven_access(monkeypatch, tmp_path) -> None:
    module = _module(monkeypatch)
    module.HEALTH_PATH = tmp_path / "claude.json"
    module.HEALTH_PATH.write_text(
        json.dumps(
            {
                "authenticated": False,
                "last_success_at": "2026-01-01T00:00:00+00:00",
            }
        ),
        encoding="utf-8",
    )
    monkeypatch.setattr(
        module.subprocess,
        "run",
        lambda *_args, **_kwargs: SimpleNamespace(
            returncode=1, stdout="", stderr="transient timeout"
        ),
    )

    health = module._subscription_health(force=True)

    assert health["authenticated"] is True
    assert health["state"] == "available"
    assert health["api_provider"] == "firstParty"
    assert health["auth_method"] == "oauth_token"
    assert health["auth_probe_stale"] is True
    assert health["last_authenticated_at"] == "2026-01-01T00:00:00+00:00"


def test_explicit_logged_out_probe_clears_recent_access(monkeypatch, tmp_path) -> None:
    module = _module(monkeypatch)
    module.HEALTH_PATH = tmp_path / "claude.json"
    module.HEALTH_PATH.write_text(
        json.dumps(
            {
                "authenticated": True,
                "api_provider": "firstParty",
                "auth_method": "oauth_token",
                "last_success_at": datetime.now(timezone.utc).isoformat(),
            }
        ),
        encoding="utf-8",
    )
    monkeypatch.setattr(
        module.subprocess,
        "run",
        lambda *_args, **_kwargs: SimpleNamespace(
            returncode=1,
            stdout=json.dumps({"loggedIn": False}),
            stderr="",
        ),
    )

    health = module._subscription_health(force=True)

    assert health["authenticated"] is False
    assert health["state"] == "unavailable"
    assert health["auth_probe_stale"] is False
    assert "last_authenticated_at" not in health