"""Routing and runtime workload contracts for Hermes chat.""" from __future__ import annotations import importlib.util import json import re import sqlite3 import sys import tomllib from pathlib import Path from types import SimpleNamespace from testing.tests.test_hermes_chat_support import ( HERMES, ROOT, _documents, ) def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): """A previous gateway lifetime cannot leave phantom active workers.""" module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" spec = importlib.util.spec_from_file_location( "close_stale_api_workers", module_path ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) database = tmp_path / "state.db" with sqlite3.connect(database) as connection: connection.execute( "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " "end_reason TEXT, archived INTEGER DEFAULT 0)" ) connection.executemany( "INSERT INTO sessions " "(id, source, parent_session_id, started_at, ended_at, end_reason) " "VALUES (?, ?, ?, 1, ?, ?)", ( ("stale", "api_server", "parent", None, None), ("root", "api_server", None, None, None), ("finished", "api_server", "parent", 2.0, "api_run_completed"), ("interactive", "tui", "parent", None, None), ), ) assert module.migrate(database) == 1 assert module.migrate(database) == 0 with sqlite3.connect(database) as connection: rows = connection.execute( "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" ).fetchall() by_id = {row[0]: row[1:] for row in rows} assert by_id["stale"][0] is not None assert by_id["stale"][1] == "api_run_recovered_stale" assert by_id["root"] == (None, None) assert by_id["finished"] == (2.0, "api_run_completed") assert by_id["interactive"] == (None, None) def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): """Thin brokers stay small while native Claude runs beside owner auth.""" dockerfile = ( ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" ).read_text() assert "httpx==0.28.1" in dockerfile assert "worker_route_broker.py" in dockerfile assert "routing_catalog.py" in dockerfile deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] containers = { container["name"]: container for container in deployment["spec"]["template"]["spec"]["containers"] } expected = ( "registry.bstein.dev/bstein/hermes-switchyard-brokers@" "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" ) assert containers["worker-route-broker"]["image"] == expected assert containers["classifier-broker"]["image"] == expected assert "claude-oauth-broker" not in containers agent = _documents(HERMES / "agent-deployment.yaml")[0] agent_containers = { container["name"]: container for container in agent["spec"]["template"]["spec"]["containers"] } for container_name in ("hermes", "terminal"): container = agent_containers[container_name] environment = {item["name"]: item["value"] for item in container["env"]} mounts = {item["name"]: item for item in container["volumeMounts"]} assert ( environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" ) assert ( environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" ) assert ( environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" ) assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" assert mounts["routing-catalog"]["readOnly"] is True codex = agent_containers["codex-broker"] codex_environment = {item["name"]: item["value"] for item in codex["env"]} assert ( codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" ) claude = agent_containers["claude-broker"] assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] assert any( mount["name"] == "home" and mount["mountPath"] == "/opt/data" for mount in claude["volumeMounts"] ) def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): """AUTO classification must fit the Jetson context without losing intent.""" broker_path = HERMES / "scripts" / "classifier_broker.py" monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) spec = importlib.util.spec_from_file_location( "hermes_classifier_broker", broker_path ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) payload = { "model": "qwen2.5:14b-instruct-q4_0", "messages": [ { "role": "system", "content": "routing contract\n" + ("candidate policy " * 1000), }, { "role": "user", "content": "Build and verify the Cassandra release safely.", }, { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_large", "type": "function", "function": { "name": "large_tool", "arguments": '{"command":"' + ("x" * 5000) + '"}', }, } ], }, { "role": "tool", "content": "unbounded test output " * 10000, "tool_call_id": "call_large", }, { "role": "user", "content": [ { "type": "image_url", "image_url": {"url": "data:image/png;base64,private"}, }, {"type": "text", "text": "Turn this cat into a cute clown."}, ], }, ], "tools": [{"type": "function", "function": {"name": "large_tool"}}], "tool_choice": "auto", "parallel_tool_calls": True, "response_format": {"type": "json_object"}, } compacted = module.compact_payload(payload) encoded = json.dumps(compacted) assert compacted["model"] == payload["model"] assert compacted["response_format"] == payload["response_format"] assert "tools" not in compacted assert "tool_choice" not in compacted assert "parallel_tool_calls" not in compacted assert all(message.get("role") != "tool" for message in compacted["messages"]) assert all("tool_call_id" not in message for message in compacted["messages"]) assert all("tool_calls" not in message for message in compacted["messages"]) assert "[tool evidence]" in encoded assert "[assistant requested an external tool]" in encoded assert "Build and verify the Cassandra release safely." in encoded assert "Turn this cat into a cute clown." in encoded assert "image attachment available to the selected worker" in encoded assert "data:image/png;base64" not in encoded assert "unbounded test output " * 100 not in encoded assert len(encoded) < 18_000 def test_switchyard_classifier_is_bounded_and_fails_open_once(): """A sick local judge must not hold chat through repeated long retries.""" config = tomllib.loads( _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] ) classifier = config["llm_clients"]["classifier"] assert classifier["base_url"] == "http://127.0.0.1:9008/v1" assert classifier["max_retries"] == 0 for route in ("auto_fast", "auto_balanced"): assert config["routes"][route]["recent_turn_window"] == 4 for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): assert config["routes"][route]["recent_turn_window"] == 6 deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] containers = { item["name"]: item for item in deployment["spec"]["template"]["spec"]["containers"] } classifier_container = containers["classifier-broker"] env = {item["name"]: item["value"] for item in classifier_container["env"]} assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" def test_image_broker_returns_bytes_and_removes_owner_cache( tmp_path: Path, monkeypatch ): """The broker must not retain a family user's generated image.""" broker_path = HERMES / "scripts" / "image_broker.py" spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) generated = tmp_path / "generated.png" generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") class Provider: def generate(self, prompt, aspect, **kwargs): assert prompt == "paint a blue sphere" assert aspect == "square" return { "success": True, "image": str(generated), "model": "gpt-image-2-high", "quality": "high", } monkeypatch.setattr(module, "_PROVIDER", Provider()) result = module._generate( { "prompt": "paint a blue sphere", "aspect_ratio": "square", "model": "gpt-image-2-high", } ) assert result["success"] is True assert result["image_b64"] assert "image" not in result assert not generated.exists() def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( monkeypatch, ): """AUTO is hosted-first while explicit local never calls the hosted lane.""" broker_path = HERMES / "scripts" / "image_broker.py" spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) calls = [] def hosted(payload, model, prompt, aspect): calls.append(("hosted", model, prompt, aspect)) return {"success": False, "error": "hosted refusal"} def local(payload, timeout=1800.0): calls.append(("local", payload["model"], timeout)) return { "success": True, "image_b64": "aW1hZ2U=", "model": "flux-2-klein-4b-local", "route": "local", } monkeypatch.setattr(module, "_generate_hosted", hosted) monkeypatch.setattr(module, "_local_request", local) auto = module._generate( { "prompt": "colorize this family photograph", "aspect_ratio": "portrait", "model": "atlas-image-auto-high", } ) assert auto["success"] is True assert auto["route"] == "local" assert auto["hosted_fallback_reason"] == "hosted refusal" assert [call[0] for call in calls] == ["hosted", "local"] calls.clear() explicit = module._generate( { "prompt": "make a local landscape", "aspect_ratio": "landscape", "model": "flux-2-klein-4b-local", } ) assert explicit["route"] == "local" assert [call[0] for call in calls] == ["local"] def test_image_broker_policy_is_narrow_and_operator_extensible( tmp_path: Path, monkeypatch ): """Family-photo restoration stays allowed while the hard boundary remains.""" broker_path = HERMES / "scripts" / "image_broker.py" spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) policy = tmp_path / "policy.json" policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') monkeypatch.setattr(module, "POLICY_PATH", policy) assert ( module._policy_error( "Colorize my baby photograph with a lighter natural skin tone" ) is None ) assert "minors" in module._policy_error("Create a sexual image of a child") assert module._policy_error("A site-specific block request") == ( "request is blocked by the operator image policy" ) def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): """FLUX and Wolf share titan-24 while text stays on titan-20.""" deployment = _documents(HERMES / "local-image-deployment.yaml")[0] assert deployment["metadata"]["name"] == "hermes-local-image" pod = deployment["spec"]["template"]["spec"] assert pod["serviceAccountName"] == "hermes-gpu-runtime" local = next(item for item in pod["containers"] if item["name"] == "local-image") assert len(pod["containers"]) == 1 assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) model_env = {item["name"]: item["value"] for item in local["env"]} assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( "e7b7dc27f91deacad38e78976d1f2b499d76a294" ) assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( "3221225472" ) assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" assert ( "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] ) models_volume = next(item for item in pod["volumes"] if item["name"] == "models") assert models_volume["persistentVolumeClaim"]["claimName"] == ( "hermes-image-models" ) services = _documents(HERMES / "service.yaml") image_service = next( item for item in services if item["metadata"]["name"] == "hermes-local-image" ) assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} handoff_services = _documents(HERMES / "model-gate-deployment.yaml") handoff = next( item for item in handoff_services if item["kind"] == "Service" and item["metadata"]["name"] == "hermes-gpu-handoff" ) assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" ariadne = _documents( Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" )[0] env = { item["name"]: item["value"] for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] if "value" in item } assert env["GAME_MODE_OLLAMA_URL"] == ( "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" ) assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" for config_name in ( "configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml", ): config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] assert "gpt-oss:20b" not in config assert "atlas-switchyard" in config switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ "routes.toml" ] assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard assert "qwen2.5:3b-instruct-q4_0" not in switchyard assert "route/local/qwen2.5-14b/medium" in switchyard assert "Anthropic and Claude name the same provider" in switchyard assert "OpenAI and Codex name the same provider" in switchyard assert "account-visible economy, balanced, advanced, or frontier" in switchyard assert "Choose across every configured Codex and Claude family" not in switchyard assert switchyard.count('Treat "think hard"') == 4 assert switchyard.count("Never choose below the") >= 5 switchyard_config = tomllib.loads(switchyard) routes = switchyard_config["routes"] configured_targets = switchyard_config["targets"] capability_rank = { "economy": 0, "balanced": 1, "advanced": 2, "frontier": 3, } def target_capability(target: str) -> str | None: parts = target.split("_auto_") if len(parts) != 2 or parts[1].startswith(("low", "medium", "high", "xhigh")): return None return parts[1].rsplit("_", 1)[0] for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): targets = routes[route_name]["targets"] capabilities = [ target_capability(target) for target in targets if target_capability(target) is not None ] # Switchyard retries the full target list after an upstream error. A # failed Astra target must therefore be considered before any Sol one. assert capabilities == sorted( capabilities, key=capability_rank.__getitem__, reverse=True ) assert "max_output_tokens" not in routes[route_name] for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): targets = routes[route_name]["targets"] selector_targets = routes[route_name]["response_schema"] assert not any(target.startswith("local_") for target in targets) assert all("_auto_" in target or target.startswith("neutral_") for target in targets) assert "local_qwen" not in selector_targets assert "not eligible for foreground" in routes[route_name]["prompt"] for route_name in ("auto_deep", "auto_maximum"): targets = routes[route_name]["targets"] selector_targets = routes[route_name]["response_schema"] assert not any(target.endswith("_low") for target in targets) assert "_low" not in selector_targets maximum_targets = routes["auto_maximum"]["targets"] maximum_selector_targets = routes["auto_maximum"]["response_schema"] assert not any(target.endswith("_medium") for target in maximum_targets) assert "_medium" not in maximum_selector_targets assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] assert "quality mark was missed" in routes["auto_balanced"]["prompt"] assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] assert ( "Repeated quality misses require xhigh" in routes["worker_auto_maximum"]["prompt"] ) worker_capabilities = [ target_capability(target) for target in routes["worker_auto_maximum"]["targets"] if target_capability(target) is not None ] assert worker_capabilities == sorted( worker_capabilities, key=capability_rank.__getitem__, reverse=True ) for route_name in ( "fallback_fast", "fallback_balanced", "fallback_deep", "fallback_maximum", "fallback_worker_maximum", ): assert all( target_capability(target) == "advanced" for target in routes[route_name]["targets"] ) assert any( target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] ) for route_name in ( "manual_codex_luna", "manual_codex_terra", "manual_codex_sol", "manual_claude_haiku", "manual_claude_fable", "manual_claude_sonnet", "manual_claude_opus", ): assert not any( target.startswith("local_") for target in routes[route_name]["targets"] ) for provider, families in { "codex": ("luna", "terra", "sol"), "claude": ("haiku", "fable", "sonnet", "opus"), }.items(): for family in families: for effort in ("low", "medium", "high", "xhigh"): route = routes[f"manual_{provider}_{family}_{effort}"] assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" assert route["targets"][0] == f"{provider}_{family}_{effort}" worker_target = f"worker_{provider}_{family}_{effort}" assert configured_targets[worker_target]["id"] == ( f"worker/{provider}/{family}/{effort}" ) assert all("_auto_" in target or target.startswith("neutral_") for target in routes["worker_auto_maximum"]["targets"]) for route_name in ("auto_fast", "auto_balanced"): prompt = routes[route_name]["prompt"] assert "image tool—not the conversational model" in prompt assert "Do not select a" in prompt assert "local Qwen or Claude target" in prompt assert "max_output_tokens" not in routes["worker_auto_maximum"] model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py" ] assert "qwen2.5:14b-instruct-q4_0" in model_gate def test_classifier_prompt_examples_are_allowed_and_schema_valid(): """Classifier prompt examples must name configured capability targets.""" switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ "routes.toml" ] routes = tomllib.loads(switchyard)["routes"] target_pattern = re.compile( r"\b(?:worker_)?(?:codex|claude)_auto_[a-z0-9_]+\b" ) classifier_count = 0 example_count = 0 for route in routes.values(): if route.get("type") != "llm_classifier": continue classifier_count += 1 examples = target_pattern.findall(route["prompt"]) example_count += len(examples) schema = json.loads(route["response_schema"]) pattern = schema["properties"]["decision"]["properties"]["target"][ "pattern" ] allowed_targets = set(route["targets"]) assert all(target in allowed_targets for target in examples), route["id"] assert all(re.fullmatch(pattern, target) for target in examples), route["id"] assert classifier_count >= 5 assert example_count >= 8