529 lines
21 KiB
Python
529 lines
21 KiB
Python
"""Routing and runtime workload contracts for Hermes chat."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
import tomllib
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
|
|
from testing.tests.test_hermes_chat_support import (
|
|
HERMES,
|
|
ROOT,
|
|
_documents,
|
|
)
|
|
|
|
|
|
def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path):
|
|
"""A previous gateway lifetime cannot leave phantom active workers."""
|
|
module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
|
|
spec = importlib.util.spec_from_file_location(
|
|
"close_stale_api_workers", module_path
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
database = tmp_path / "state.db"
|
|
with sqlite3.connect(database) as connection:
|
|
connection.execute(
|
|
"CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
|
|
"parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, "
|
|
"end_reason TEXT, archived INTEGER DEFAULT 0)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO sessions "
|
|
"(id, source, parent_session_id, started_at, ended_at, end_reason) "
|
|
"VALUES (?, ?, ?, 1, ?, ?)",
|
|
(
|
|
("stale", "api_server", "parent", None, None),
|
|
("root", "api_server", None, None, None),
|
|
("finished", "api_server", "parent", 2.0, "api_run_completed"),
|
|
("interactive", "tui", "parent", None, None),
|
|
),
|
|
)
|
|
|
|
assert module.migrate(database) == 1
|
|
assert module.migrate(database) == 0
|
|
with sqlite3.connect(database) as connection:
|
|
rows = connection.execute(
|
|
"SELECT id, ended_at, end_reason FROM sessions ORDER BY id"
|
|
).fetchall()
|
|
by_id = {row[0]: row[1:] for row in rows}
|
|
assert by_id["stale"][0] is not None
|
|
assert by_id["stale"][1] == "api_run_recovered_stale"
|
|
assert by_id["root"] == (None, None)
|
|
assert by_id["finished"] == (2.0, "api_run_completed")
|
|
assert by_id["interactive"] == (None, None)
|
|
|
|
|
|
def test_switchyard_brokers_and_native_claude_lane_use_the_right_images():
|
|
"""Thin brokers stay small while native Claude runs beside owner auth."""
|
|
dockerfile = (
|
|
ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers"
|
|
).read_text()
|
|
assert "httpx==0.28.1" in dockerfile
|
|
assert "worker_route_broker.py" in dockerfile
|
|
assert "routing_catalog.py" in dockerfile
|
|
|
|
deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
|
|
containers = {
|
|
container["name"]: container
|
|
for container in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
expected = (
|
|
"registry.bstein.dev/bstein/hermes-switchyard-brokers@"
|
|
"sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083"
|
|
)
|
|
assert containers["worker-route-broker"]["image"] == expected
|
|
assert containers["classifier-broker"]["image"] == expected
|
|
assert "claude-oauth-broker" not in containers
|
|
|
|
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
|
agent_containers = {
|
|
container["name"]: container
|
|
for container in agent["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
for container_name in ("hermes", "terminal"):
|
|
container = agent_containers[container_name]
|
|
environment = {item["name"]: item["value"] for item in container["env"]}
|
|
mounts = {item["name"]: item for item in container["volumeMounts"]}
|
|
assert (
|
|
environment["HERMES_ROUTING_CATALOG_PATH"]
|
|
== "/routing-catalog/catalog.json"
|
|
)
|
|
assert (
|
|
environment["HERMES_CODEX_HEALTH_PATH"]
|
|
== "/opt/data/provider-health/codex.json"
|
|
)
|
|
assert (
|
|
environment["HERMES_CLAUDE_HEALTH_PATH"]
|
|
== "/opt/data/provider-health/claude.json"
|
|
)
|
|
assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog"
|
|
assert mounts["routing-catalog"]["readOnly"] is True
|
|
codex = agent_containers["codex-broker"]
|
|
codex_environment = {item["name"]: item["value"] for item in codex["env"]}
|
|
assert (
|
|
codex_environment["HERMES_CODEX_HEALTH_PATH"]
|
|
== "/opt/data/provider-health/codex.json"
|
|
)
|
|
claude = agent_containers["claude-broker"]
|
|
assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@")
|
|
assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0]
|
|
assert any(
|
|
mount["name"] == "home" and mount["mountPath"] == "/opt/data"
|
|
for mount in claude["volumeMounts"]
|
|
)
|
|
|
|
|
|
def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch):
|
|
"""AUTO classification must fit the Jetson context without losing intent."""
|
|
broker_path = HERMES / "scripts" / "classifier_broker.py"
|
|
monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
|
|
spec = importlib.util.spec_from_file_location(
|
|
"hermes_classifier_broker", broker_path
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
payload = {
|
|
"model": "qwen2.5:14b-instruct-q4_0",
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": "routing contract\n" + ("candidate policy " * 1000),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "Build and verify the Cassandra release safely.",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_large",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "large_tool",
|
|
"arguments": '{"command":"' + ("x" * 5000) + '"}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": "unbounded test output " * 10000,
|
|
"tool_call_id": "call_large",
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": "data:image/png;base64,private"},
|
|
},
|
|
{"type": "text", "text": "Turn this cat into a cute clown."},
|
|
],
|
|
},
|
|
],
|
|
"tools": [{"type": "function", "function": {"name": "large_tool"}}],
|
|
"tool_choice": "auto",
|
|
"parallel_tool_calls": True,
|
|
"response_format": {"type": "json_object"},
|
|
}
|
|
|
|
compacted = module.compact_payload(payload)
|
|
encoded = json.dumps(compacted)
|
|
|
|
assert compacted["model"] == payload["model"]
|
|
assert compacted["response_format"] == payload["response_format"]
|
|
assert "tools" not in compacted
|
|
assert "tool_choice" not in compacted
|
|
assert "parallel_tool_calls" not in compacted
|
|
assert all(message.get("role") != "tool" for message in compacted["messages"])
|
|
assert all("tool_call_id" not in message for message in compacted["messages"])
|
|
assert all("tool_calls" not in message for message in compacted["messages"])
|
|
assert "[tool evidence]" in encoded
|
|
assert "[assistant requested an external tool]" in encoded
|
|
assert "Build and verify the Cassandra release safely." in encoded
|
|
assert "Turn this cat into a cute clown." in encoded
|
|
assert "image attachment available to the selected worker" in encoded
|
|
assert "data:image/png;base64" not in encoded
|
|
assert "unbounded test output " * 100 not in encoded
|
|
assert len(encoded) < 18_000
|
|
|
|
|
|
def test_switchyard_classifier_is_bounded_and_fails_open_once():
|
|
"""A sick local judge must not hold chat through repeated long retries."""
|
|
config = tomllib.loads(
|
|
_documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"]
|
|
)
|
|
classifier = config["llm_clients"]["classifier"]
|
|
assert classifier["base_url"] == "http://127.0.0.1:9008/v1"
|
|
assert classifier["max_retries"] == 0
|
|
for route in ("auto_fast", "auto_balanced"):
|
|
assert config["routes"][route]["recent_turn_window"] == 4
|
|
for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"):
|
|
assert config["routes"][route]["recent_turn_window"] == 6
|
|
|
|
deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
|
|
containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
classifier_container = containers["classifier-broker"]
|
|
env = {item["name"]: item["value"] for item in classifier_container["env"]}
|
|
assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60"
|
|
assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier"
|
|
|
|
|
|
def test_image_broker_returns_bytes_and_removes_owner_cache(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
"""The broker must not retain a family user's generated image."""
|
|
broker_path = HERMES / "scripts" / "image_broker.py"
|
|
spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
generated = tmp_path / "generated.png"
|
|
generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image")
|
|
|
|
class Provider:
|
|
def generate(self, prompt, aspect, **kwargs):
|
|
assert prompt == "paint a blue sphere"
|
|
assert aspect == "square"
|
|
return {
|
|
"success": True,
|
|
"image": str(generated),
|
|
"model": "gpt-image-2-high",
|
|
"quality": "high",
|
|
}
|
|
|
|
monkeypatch.setattr(module, "_PROVIDER", Provider())
|
|
result = module._generate(
|
|
{
|
|
"prompt": "paint a blue sphere",
|
|
"aspect_ratio": "square",
|
|
"model": "gpt-image-2-high",
|
|
}
|
|
)
|
|
|
|
assert result["success"] is True
|
|
assert result["image_b64"]
|
|
assert "image" not in result
|
|
assert not generated.exists()
|
|
|
|
|
|
def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes(
|
|
monkeypatch,
|
|
):
|
|
"""AUTO is hosted-first while explicit local never calls the hosted lane."""
|
|
broker_path = HERMES / "scripts" / "image_broker.py"
|
|
spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
calls = []
|
|
|
|
def hosted(payload, model, prompt, aspect):
|
|
calls.append(("hosted", model, prompt, aspect))
|
|
return {"success": False, "error": "hosted refusal"}
|
|
|
|
def local(payload, timeout=1800.0):
|
|
calls.append(("local", payload["model"], timeout))
|
|
return {
|
|
"success": True,
|
|
"image_b64": "aW1hZ2U=",
|
|
"model": "flux-2-klein-4b-local",
|
|
"route": "local",
|
|
}
|
|
|
|
monkeypatch.setattr(module, "_generate_hosted", hosted)
|
|
monkeypatch.setattr(module, "_local_request", local)
|
|
auto = module._generate(
|
|
{
|
|
"prompt": "colorize this family photograph",
|
|
"aspect_ratio": "portrait",
|
|
"model": "atlas-image-auto-high",
|
|
}
|
|
)
|
|
assert auto["success"] is True
|
|
assert auto["route"] == "local"
|
|
assert auto["hosted_fallback_reason"] == "hosted refusal"
|
|
assert [call[0] for call in calls] == ["hosted", "local"]
|
|
|
|
calls.clear()
|
|
explicit = module._generate(
|
|
{
|
|
"prompt": "make a local landscape",
|
|
"aspect_ratio": "landscape",
|
|
"model": "flux-2-klein-4b-local",
|
|
}
|
|
)
|
|
assert explicit["route"] == "local"
|
|
assert [call[0] for call in calls] == ["local"]
|
|
|
|
|
|
def test_image_broker_policy_is_narrow_and_operator_extensible(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
"""Family-photo restoration stays allowed while the hard boundary remains."""
|
|
broker_path = HERMES / "scripts" / "image_broker.py"
|
|
spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
policy = tmp_path / "policy.json"
|
|
policy.write_text('{"additional_blocked_phrases":["site-specific block"]}')
|
|
monkeypatch.setattr(module, "POLICY_PATH", policy)
|
|
|
|
assert (
|
|
module._policy_error(
|
|
"Colorize my baby photograph with a lighter natural skin tone"
|
|
)
|
|
is None
|
|
)
|
|
assert "minors" in module._policy_error("Create a sexual image of a child")
|
|
assert module._policy_error("A site-specific block request") == (
|
|
"request is blocked by the operator image policy"
|
|
)
|
|
|
|
|
|
def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|
"""FLUX and Wolf share titan-24 while text stays on titan-20."""
|
|
deployment = _documents(HERMES / "local-image-deployment.yaml")[0]
|
|
assert deployment["metadata"]["name"] == "hermes-local-image"
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
assert pod["serviceAccountName"] == "hermes-gpu-runtime"
|
|
local = next(item for item in pod["containers"] if item["name"] == "local-image")
|
|
assert len(pod["containers"]) == 1
|
|
assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1
|
|
assert local["ports"] == [{"name": "local-image", "containerPort": 9004}]
|
|
assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"])
|
|
model_env = {item["name"]: item["value"] for item in local["env"]}
|
|
assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004"
|
|
assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
|
|
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
|
|
)
|
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
|
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
|
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
|
"3221225472"
|
|
)
|
|
assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential"
|
|
assert (
|
|
"nvidia-process-exporter-local.monitoring.svc.cluster.local"
|
|
in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"]
|
|
)
|
|
models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
|
|
assert models_volume["persistentVolumeClaim"]["claimName"] == (
|
|
"hermes-image-models"
|
|
)
|
|
|
|
services = _documents(HERMES / "service.yaml")
|
|
image_service = next(
|
|
item for item in services if item["metadata"]["name"] == "hermes-local-image"
|
|
)
|
|
assert image_service["spec"]["selector"] == {"app": "hermes-local-image"}
|
|
|
|
handoff_services = _documents(HERMES / "model-gate-deployment.yaml")
|
|
handoff = next(
|
|
item
|
|
for item in handoff_services
|
|
if item["kind"] == "Service"
|
|
and item["metadata"]["name"] == "hermes-gpu-handoff"
|
|
)
|
|
assert handoff["spec"]["ports"][0]["targetPort"] == "handoff"
|
|
|
|
ariadne = _documents(
|
|
Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml"
|
|
)[0]
|
|
env = {
|
|
item["name"]: item["value"]
|
|
for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"]
|
|
if "value" in item
|
|
}
|
|
assert env["GAME_MODE_OLLAMA_URL"] == (
|
|
"http://hermes-gpu-handoff.hermes.svc.cluster.local:11434"
|
|
)
|
|
assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local"
|
|
|
|
for config_name in (
|
|
"configmap.yaml",
|
|
"agent-configmap.yaml",
|
|
"chat-configmap.yaml",
|
|
):
|
|
config = _documents(HERMES / config_name)[0]["data"]["config.yaml"]
|
|
assert "gpt-oss:20b" not in config
|
|
assert "atlas-switchyard" in config
|
|
switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][
|
|
"routes.toml"
|
|
]
|
|
assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard
|
|
assert "qwen2.5:3b-instruct-q4_0" not in switchyard
|
|
assert "route/local/qwen2.5-14b/medium" in switchyard
|
|
assert "Anthropic and Claude name the same provider" in switchyard
|
|
assert "OpenAI and Codex name the same provider" in switchyard
|
|
assert "account-visible economy, balanced, or advanced" in switchyard
|
|
assert "Choose across every configured Codex and Claude family" not in switchyard
|
|
assert switchyard.count('Treat "think hard"') == 4
|
|
assert switchyard.count("Never choose below the") >= 5
|
|
switchyard_config = tomllib.loads(switchyard)
|
|
routes = switchyard_config["routes"]
|
|
configured_targets = switchyard_config["targets"]
|
|
capability_rank = {
|
|
"economy": 0,
|
|
"balanced": 1,
|
|
"advanced": 2,
|
|
"frontier": 3,
|
|
}
|
|
|
|
def target_capability(target: str) -> str | None:
|
|
parts = target.split("_auto_")
|
|
if len(parts) != 2 or parts[1].startswith(("low", "medium", "high", "xhigh")):
|
|
return None
|
|
return parts[1].rsplit("_", 1)[0]
|
|
|
|
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
|
targets = routes[route_name]["targets"]
|
|
capabilities = [
|
|
target_capability(target)
|
|
for target in targets
|
|
if target_capability(target) is not None
|
|
]
|
|
# Switchyard retries the full target list after an upstream error. A
|
|
# failed Astra target must therefore be considered before any Sol one.
|
|
assert capabilities == sorted(
|
|
capabilities, key=capability_rank.__getitem__, reverse=True
|
|
)
|
|
assert "max_output_tokens" not in routes[route_name]
|
|
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
|
targets = routes[route_name]["targets"]
|
|
selector_targets = routes[route_name]["response_schema"]
|
|
assert not any(target.startswith("local_") for target in targets)
|
|
assert all("_auto_" in target or target.startswith("neutral_") for target in targets)
|
|
assert "local_qwen" not in selector_targets
|
|
assert "not eligible for foreground" in routes[route_name]["prompt"]
|
|
for route_name in ("auto_deep", "auto_maximum"):
|
|
targets = routes[route_name]["targets"]
|
|
selector_targets = routes[route_name]["response_schema"]
|
|
assert not any(target.endswith("_low") for target in targets)
|
|
assert "_low" not in selector_targets
|
|
maximum_targets = routes["auto_maximum"]["targets"]
|
|
maximum_selector_targets = routes["auto_maximum"]["response_schema"]
|
|
assert not any(target.endswith("_medium") for target in maximum_targets)
|
|
assert "_medium" not in maximum_selector_targets
|
|
assert "absolute high effort floor" in routes["auto_maximum"]["prompt"]
|
|
assert "quality mark was missed" in routes["auto_balanced"]["prompt"]
|
|
assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"]
|
|
assert (
|
|
"Repeated quality misses require xhigh"
|
|
in routes["worker_auto_maximum"]["prompt"]
|
|
)
|
|
worker_capabilities = [
|
|
target_capability(target)
|
|
for target in routes["worker_auto_maximum"]["targets"]
|
|
if target_capability(target) is not None
|
|
]
|
|
assert worker_capabilities == sorted(
|
|
worker_capabilities, key=capability_rank.__getitem__, reverse=True
|
|
)
|
|
for route_name in (
|
|
"fallback_fast",
|
|
"fallback_balanced",
|
|
"fallback_deep",
|
|
"fallback_maximum",
|
|
"fallback_worker_maximum",
|
|
):
|
|
assert all(
|
|
target_capability(target) == "advanced"
|
|
for target in routes[route_name]["targets"]
|
|
)
|
|
assert any(
|
|
target.startswith("local_") for target in routes["manual_local_qwen"]["targets"]
|
|
)
|
|
for route_name in (
|
|
"manual_codex_luna",
|
|
"manual_codex_terra",
|
|
"manual_codex_sol",
|
|
"manual_claude_haiku",
|
|
"manual_claude_fable",
|
|
"manual_claude_sonnet",
|
|
"manual_claude_opus",
|
|
):
|
|
assert not any(
|
|
target.startswith("local_") for target in routes[route_name]["targets"]
|
|
)
|
|
for provider, families in {
|
|
"codex": ("luna", "terra", "sol"),
|
|
"claude": ("haiku", "fable", "sonnet", "opus"),
|
|
}.items():
|
|
for family in families:
|
|
for effort in ("low", "medium", "high", "xhigh"):
|
|
route = routes[f"manual_{provider}_{family}_{effort}"]
|
|
assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
|
|
assert route["targets"][0] == f"{provider}_{family}_{effort}"
|
|
worker_target = f"worker_{provider}_{family}_{effort}"
|
|
assert configured_targets[worker_target]["id"] == (
|
|
f"worker/{provider}/{family}/{effort}"
|
|
)
|
|
assert all("_auto_" in target or target.startswith("neutral_")
|
|
for target in routes["worker_auto_maximum"]["targets"])
|
|
for route_name in ("auto_fast", "auto_balanced"):
|
|
prompt = routes[route_name]["prompt"]
|
|
assert "image tool—not the conversational model" in prompt
|
|
assert "Do not select a" in prompt
|
|
assert "local Qwen or Claude target" in prompt
|
|
assert "max_output_tokens" not in routes["worker_auto_maximum"]
|
|
model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
|
|
"model_gate.py"
|
|
]
|
|
assert "qwen2.5:14b-instruct-q4_0" in model_gate
|