"""Unit tests for Hermes coordinator model routing and profile generation.""" from __future__ import annotations import importlib.util import json import subprocess import sys import tomllib from pathlib import Path import pytest import yaml SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" sys.path.insert(0, str(SCRIPT.parent)) routing = importlib.import_module("hermes_model_routing") catalog_resolver = importlib.import_module("routing_catalog") SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) assert SPEC and SPEC.loader coordinator = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = coordinator SPEC.loader.exec_module(coordinator) def _base_config() -> dict: """Return the minimal coordinator configuration used by routing tests.""" return { "model": { "provider": "openai-codex", "default": "gpt-5.6-terra", "model": "gpt-5.6-terra", }, "fallback_providers": [ {"provider": "anthropic", "model": "claude-opus-5"}, dict(routing.LOCAL_FALLBACK), ], "toolsets": ["kanban"], "terminal": {"cwd": "/opt/data/workspace"}, } def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) assert routing.model_version("gpt-5.7-terra") == (5, 7) codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"] assert routing.choose_codex_model(codex) == "gpt-5.6-sol" assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra" assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.7-terra" claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"] assert routing.choose_claude_model(claude) == "claude-sonnet-5" assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna" assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra" assert routing.choose_codex_for_effort(codex, "xhigh") == "gpt-5.6-sol" assert routing.choose_claude_for_effort(claude, "low") == "claude-haiku-5" assert routing.choose_claude_for_effort(claude, "medium") == "claude-sonnet-5" assert routing.choose_claude_for_effort(claude, "xhigh") == "claude-opus-4.8" def test_empty_catalog_retains_current_models(): assert routing.choose_codex_model([], "gpt-5.6-terra") == "gpt-5.6-terra" assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5" def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): """Stable Switchyard aliases follow live releases and survive catalog outages.""" codex = routing.Catalog( "openai-codex", ["gpt-5.7-luna", "gpt-5.7-terra", "gpt-5.7-sol"], True, True, "connected", ) claude = routing.Catalog( "anthropic", ["claude-haiku-5", "claude-sonnet-6", "claude-opus-6"], True, True, "connected", ) current = routing.build_routing_catalog(codex, claude) assert ( catalog_resolver.resolve_model("codex", "auto", "high", current) == "gpt-5.7-sol" ) assert ( catalog_resolver.resolve_model("codex", "terra", "medium", current) == "gpt-5.7-terra" ) assert ( catalog_resolver.resolve_model("claude", "auto", "xhigh", current) == "claude-opus-6" ) assert ( catalog_resolver.resolve_model("claude", "sonnet", "high", current) == "claude-sonnet-6" ) unavailable = routing.Catalog("openai-codex", [], False, True, "degraded") preserved = routing.build_routing_catalog(unavailable, unavailable, current) assert ( preserved["providers"]["codex"]["resolved"] == current["providers"]["codex"]["resolved"] ) assert ( preserved["providers"]["claude"]["tiers"] == current["providers"]["claude"]["tiers"] ) def test_live_catalog_never_routes_to_a_removed_model_tier(): """A live provider catalog must replace a retired tier with a live model.""" previous = { "providers": { "codex": { "models": ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"], "resolved": { "low": "gpt-5.6-luna", "medium": "gpt-5.6-terra", "high": "gpt-5.6-sol", "xhigh": "gpt-5.6-sol", }, "tiers": { "luna": "gpt-5.6-luna", "terra": "gpt-5.6-terra", "sol": "gpt-5.6-sol", }, } } } codex = routing.Catalog( "openai-codex", ["gpt-5.7-terra", "gpt-5.7-sol"], True, True, "connected" ) claude = routing.Catalog("anthropic", ["claude-sonnet-6"], True, True, "connected") current = routing.build_routing_catalog(codex, claude, previous) assert current["providers"]["codex"]["tiers"]["luna"] == "gpt-5.7-terra" assert current["providers"]["codex"]["tiers"]["luna"] in codex.models assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6" def test_switchyard_targets_have_unique_upstream_identities(): """Switchyard drops duplicate client/model pairs, so reject them in Git.""" manifest = yaml.safe_load( (SCRIPT.parents[1] / "switchyard-configmap.yaml").read_text(encoding="utf-8") ) config = tomllib.loads(manifest["data"]["routes.toml"]) target_names = set(config["targets"]) identities: set[tuple[str, str]] = set() for target in config["targets"].values(): identity = (target["llm_client"], target["id"]) assert identity not in identities identities.add(identity) for route in config["routes"].values(): for target_name in route.get("targets", []): assert target_name in target_names if route.get("target"): assert route["target"] in target_names def test_worker_alias_resolves_before_cli_launch(): document = { "providers": { "codex": { "resolved": {"high": "gpt-5.8-sol"}, "tiers": {"sol": "gpt-5.8-sol"}, } } } assert ( catalog_resolver.resolve_worker_route("worker/codex/auto/high", document) == "worker/codex/gpt-5.8-sol/high" ) def test_codex_cli_login_counts_as_connected_runtime(monkeypatch): """AUTO routing must recognize the authenticated app-server CLI lane.""" monkeypatch.setattr(routing.shutil, "which", lambda name: "/usr/bin/codex") monkeypatch.setattr( routing.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess( args[0], 0, stdout="Logged in using ChatGPT\n", stderr="", ), ) assert routing.codex_cli_authenticated() is True def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): (tmp_path / "config.yaml").write_text( yaml.safe_dump(_base_config()), encoding="utf-8" ) (tmp_path / ".env").write_text( "API_SERVER_KEY=keep-root-only\n" "CLAUDE_CODE_OAUTH_TOKEN=claude-secret\n" "GITEA_TOKEN=gitea-secret\n" "GIT_ASKPASS=/opt/coordinator/gitea_askpass.sh\n", encoding="utf-8", ) stale_profile = tmp_path / "profiles/codex-high" stale_profile.mkdir(parents=True) (stale_profile / ".env").write_text( "CLAUDE_CODE_OAUTH_TOKEN=stale-claude\n" "GITEA_TOKEN=stale-gitea\n" "HERMES_IMAGE_BROKER_KEY=stale-relay\n" "USER_SETTING=preserve\n", encoding="utf-8", ) codex = routing.Catalog( "openai-codex", ["gpt-5.6-sol", "gpt-5.6-terra"], True, True, "connected" ) claude = routing.Catalog( "anthropic", ["claude-sonnet-5", "claude-opus-5"], True, True, "connected" ) routes = routing.configure_routes(tmp_path, codex, claude) root = yaml.safe_load((tmp_path / "config.yaml").read_text(encoding="utf-8")) codex_profile = yaml.safe_load( (tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8") ) codex_xhigh_profile = yaml.safe_load( (tmp_path / "profiles/codex-xhigh/config.yaml").read_text(encoding="utf-8") ) claude_profile = yaml.safe_load( (tmp_path / "profiles/claude-high/config.yaml").read_text(encoding="utf-8") ) assert root["model"] == { "provider": "atlas-switchyard", "default": "atlas/auto/maximum", "model": "atlas/auto/maximum", } assert root["fallback_providers"] == [] assert root["toolsets"] == ["kanban"] assert codex_profile["model"]["model"] == "atlas/manual/codex/sol" assert codex_profile["model"]["provider"] == "atlas-switchyard" assert claude_profile["model"]["model"] == "atlas/manual/claude/sonnet" assert claude_profile["model"]["provider"] == "atlas-switchyard" assert codex_profile["fallback_providers"] == [] assert claude_profile["fallback_providers"] == [] assert codex_profile["toolsets"] == [] assert codex_profile["agent"]["reasoning_effort"] == "high" assert codex_xhigh_profile["fallback_providers"] == [] assert routes["codex-xhigh"] == ["atlas/manual/codex/sol"] assert routes["claude-xhigh"] == ["atlas/manual/claude/opus"] assert routes["synthesis-xhigh"] == ["atlas/auto/maximum"] assert routes["coordinator"] == ["atlas/auto/maximum"] assert all( "max" not in profile_name for profile_name in routes if profile_name != "catalog" ) profile_env = (tmp_path / "profiles/codex-high/.env").read_text(encoding="utf-8") assert "CLAUDE_CODE_OAUTH_TOKEN" not in profile_env assert "GITEA_TOKEN" not in profile_env assert "HERMES_IMAGE_BROKER_KEY" not in profile_env assert "API_SERVER_KEY" not in profile_env assert "USER_SETTING=preserve" in profile_env assert "claude-secret" not in json.dumps(routes) assert (tmp_path / "profiles/codex-high/.env").stat().st_mode & 0o777 == 0o600 def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path): base = _base_config() base["model"]["model"] = base["model"]["default"] = "gpt-5.6-sol" (tmp_path / "config.yaml").write_text(yaml.safe_dump(base), encoding="utf-8") (tmp_path / ".env").write_text("", encoding="utf-8") codex = routing.Catalog("openai-codex", ["gpt-5.4"], False, True, "degraded") claude = routing.Catalog("anthropic", ["claude-haiku-4.5"], False, True, "degraded") routing.configure_routes(tmp_path, codex, claude) current = yaml.safe_load((tmp_path / "config.yaml").read_text(encoding="utf-8")) assert current["model"]["model"] == "atlas/auto/maximum" assert current["model"]["provider"] == "atlas-switchyard" assert current["fallback_providers"] == [] def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path): """A catalog outage must not bypass a managed worker's Switchyard route.""" (tmp_path / "config.yaml").write_text( yaml.safe_dump(_base_config()), encoding="utf-8" ) (tmp_path / ".env").write_text("", encoding="utf-8") live_codex = routing.Catalog( "openai-codex", ["gpt-5.6-sol", "gpt-5.6-terra"], True, True, "connected" ) live_claude = routing.Catalog( "anthropic", ["claude-opus-5"], True, True, "connected" ) routing.configure_routes(tmp_path, live_codex, live_claude) degraded_codex = routing.Catalog( "openai-codex", ["gpt-5.4"], False, True, "degraded" ) routing.configure_routes(tmp_path, degraded_codex, live_claude) worker = yaml.safe_load( (tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8") ) assert worker["model"]["model"] == "atlas/manual/codex/sol" assert worker["model"]["provider"] == "atlas-switchyard" def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): (tmp_path / "config.yaml").write_text( yaml.safe_dump(_base_config()), encoding="utf-8" ) (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) monkeypatch.setattr( coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} ) monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") status = coordinator.refresh_once(tmp_path) status_path = tmp_path / "workspace/coordinator/model-routing.json" assert status_path.is_file() assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( tmp_path: Path, monkeypatch ): """A damaged project board degrades Kanban instead of stopping all services.""" class FakeKanbanDbCorruptError(Exception): def __init__(self): super().__init__("integrity check failed") self.db_path = tmp_path / "kanban.db" self.backup_path = tmp_path / "kanban.db.corrupt.backup" self.reason = "row out of order" def raise_corruption(_root: Path) -> None: raise FakeKanbanDbCorruptError() monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) monkeypatch.setattr( coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError ) status = coordinator.bootstrap_cassandra_state(tmp_path) assert status == { "state": "corrupt-preserved", "database": str(tmp_path / "kanban.db"), "backup": str(tmp_path / "kanban.db.corrupt.backup"), "reason": "row out of order", } def test_unrelated_cassandra_bootstrap_failure_remains_fatal( tmp_path: Path, monkeypatch ): """Only the known fail-closed corruption state may be degraded.""" class FakeKanbanDbCorruptError(Exception): pass def raise_permission_error(_root: Path) -> None: raise PermissionError("cannot access board") monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) monkeypatch.setattr( coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError ) try: coordinator.bootstrap_cassandra_state(tmp_path) except PermissionError as error: assert str(error) == "cannot access board" else: raise AssertionError("unrelated board errors must fail the coordinator refresh") def test_cassandra_workspace_uses_valid_configured_worktree( tmp_path: Path, monkeypatch ): """The coordinator selects an explicit Git worktree but rejects stale paths.""" active = tmp_path / "cassandra-v69" active.mkdir() (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) assert coordinator.cassandra_workspace() == active (active / ".git").unlink() assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH @pytest.mark.parametrize( ("current_remote", "repair_action"), [ ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), (None, "add"), (coordinator.CASSANDRA_REMOTE, None), ], ids=["old-origin", "missing-origin", "canonical-origin"], ) def test_cassandra_sync_repairs_existing_worktree( tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None ): """Existing worktrees use the canonical Atlas origin before fetching.""" workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") token_path = tmp_path / "gitea-token" token_path.write_text("configured\n", encoding="utf-8") monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) commands: list[list[str]] = [] environments: list[dict[str, str]] = [] def run(command, **kwargs): commands.append(command) environments.append(kwargs.get("env", {})) if command[-2:] == ["get-url", "origin"]: return subprocess.CompletedProcess( command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" ) return subprocess.CompletedProcess(command, 0) monkeypatch.setattr(coordinator.subprocess, "run", run) assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" repair_commands = [command for command in commands if repair_action in command] assert bool(repair_commands) is bool(repair_action) assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] assert all("GITEA_TOKEN" not in environment for environment in environments) def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): """Missing credentials skip only the fetch, not the local origin repair.""" workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") commands: list[list[str]] = [] def run(command, **_kwargs): commands.append(command) stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None return subprocess.CompletedProcess(command, 0, stdout=stdout) monkeypatch.setattr(coordinator.subprocess, "run", run) state = coordinator.sync_cassandra_repo({}) assert state == "ready; fetch skipped until Gitea token is configured" assert [command[-3] for command in commands] == ["remote", "set-url"] def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" class Task: def __init__(self, task_id: str, status: str): self.id = task_id self.status = status self.workspace_kind = "dir" self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) class Closing: def __enter__(self): return object() def __exit__(self, *_args): return None class FakeKanban: tasks = [ Task("queued", "todo"), Task("active", "running"), Task("done", "done"), ] moved: list[tuple[str, str]] = [] @staticmethod def connect_closing(*, board: str): assert board == "cassandra" return Closing() @classmethod def list_tasks(cls, _connection, *, include_archived: bool): assert include_archived is False return cls.tasks @classmethod def set_workspace_path(cls, _connection, task_id: str, path: Path): cls.moved.append((task_id, str(path))) active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") coordinator._migrate_open_cassandra_tasks(FakeKanban, active) assert FakeKanban.moved == [("queued", str(active))]