atlas-iac/testing/tests/test_hermes_chat_routing_runtime.py

"""Routing and runtime workload contracts for Hermes chat."""

from __future__ import annotations

import importlib.util
import json
import sqlite3
import sys
import tomllib
from pathlib import Path
from types import SimpleNamespace


from testing.tests.test_hermes_chat_support import (
    HERMES,
    ROOT,
    _documents,
)


def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path):
    """A previous gateway lifetime cannot leave phantom active workers."""
    module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
    spec = importlib.util.spec_from_file_location(
        "close_stale_api_workers", module_path
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    database = tmp_path / "state.db"
    with sqlite3.connect(database) as connection:
        connection.execute(
            "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
            "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, "
            "end_reason TEXT, archived INTEGER DEFAULT 0)"
        )
        connection.executemany(
            "INSERT INTO sessions "
            "(id, source, parent_session_id, started_at, ended_at, end_reason) "
            "VALUES (?, ?, ?, 1, ?, ?)",
            (
                ("stale", "api_server", "parent", None, None),
                ("root", "api_server", None, None, None),
                ("finished", "api_server", "parent", 2.0, "api_run_completed"),
                ("interactive", "tui", "parent", None, None),
            ),
        )

    assert module.migrate(database) == 1
    assert module.migrate(database) == 0
    with sqlite3.connect(database) as connection:
        rows = connection.execute(
            "SELECT id, ended_at, end_reason FROM sessions ORDER BY id"
        ).fetchall()
    by_id = {row[0]: row[1:] for row in rows}
    assert by_id["stale"][0] is not None
    assert by_id["stale"][1] == "api_run_recovered_stale"
    assert by_id["root"] == (None, None)
    assert by_id["finished"] == (2.0, "api_run_completed")
    assert by_id["interactive"] == (None, None)


def test_switchyard_brokers_and_native_claude_lane_use_the_right_images():
    """Thin brokers stay small while native Claude runs beside owner auth."""
    dockerfile = (
        ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers"
    ).read_text()
    assert "httpx==0.28.1" in dockerfile
    assert "worker_route_broker.py" in dockerfile
    assert "routing_catalog.py" in dockerfile

    deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
    containers = {
        container["name"]: container
        for container in deployment["spec"]["template"]["spec"]["containers"]
    }
    expected = (
        "registry.bstein.dev/bstein/hermes-switchyard-brokers@"
        "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083"
    )
    assert containers["worker-route-broker"]["image"] == expected
    assert containers["classifier-broker"]["image"] == expected
    assert "claude-oauth-broker" not in containers

    agent = _documents(HERMES / "agent-deployment.yaml")[0]
    agent_containers = {
        container["name"]: container
        for container in agent["spec"]["template"]["spec"]["containers"]
    }
    for container_name in ("hermes", "terminal"):
        container = agent_containers[container_name]
        environment = {item["name"]: item["value"] for item in container["env"]}
        mounts = {item["name"]: item for item in container["volumeMounts"]}
        assert (
            environment["HERMES_ROUTING_CATALOG_PATH"]
            == "/routing-catalog/catalog.json"
        )
        assert (
            environment["HERMES_CODEX_HEALTH_PATH"]
            == "/opt/data/provider-health/codex.json"
        )
        assert (
            environment["HERMES_CLAUDE_HEALTH_PATH"]
            == "/opt/data/provider-health/claude.json"
        )
        assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog"
        assert mounts["routing-catalog"]["readOnly"] is True
    codex = agent_containers["codex-broker"]
    codex_environment = {item["name"]: item["value"] for item in codex["env"]}
    assert (
        codex_environment["HERMES_CODEX_HEALTH_PATH"]
        == "/opt/data/provider-health/codex.json"
    )
    claude = agent_containers["claude-broker"]
    assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@")
    assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0]
    assert any(
        mount["name"] == "home" and mount["mountPath"] == "/opt/data"
        for mount in claude["volumeMounts"]
    )


def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch):
    """AUTO classification must fit the Jetson context without losing intent."""
    broker_path = HERMES / "scripts" / "classifier_broker.py"
    monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
    spec = importlib.util.spec_from_file_location(
        "hermes_classifier_broker", broker_path
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)

    payload = {
        "model": "qwen2.5:14b-instruct-q4_0",
        "messages": [
            {
                "role": "system",
                "content": "routing contract\n" + ("candidate policy " * 1000),
            },
            {
                "role": "user",
                "content": "Build and verify the Cassandra release safely.",
            },
            {
                "role": "assistant",
                "content": None,
                "tool_calls": [
                    {
                        "id": "call_large",
                        "type": "function",
                        "function": {
                            "name": "large_tool",
                            "arguments": '{"command":"' + ("x" * 5000) + '"}',
                        },
                    }
                ],
            },
            {
                "role": "tool",
                "content": "unbounded test output " * 10000,
                "tool_call_id": "call_large",
            },
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": "data:image/png;base64,private"},
                    },
                    {"type": "text", "text": "Turn this cat into a cute clown."},
                ],
            },
        ],
        "tools": [{"type": "function", "function": {"name": "large_tool"}}],
        "tool_choice": "auto",
        "parallel_tool_calls": True,
        "response_format": {"type": "json_object"},
    }

    compacted = module.compact_payload(payload)
    encoded = json.dumps(compacted)

    assert compacted["model"] == payload["model"]
    assert compacted["response_format"] == payload["response_format"]
    assert "tools" not in compacted
    assert "tool_choice" not in compacted
    assert "parallel_tool_calls" not in compacted
    assert all(message.get("role") != "tool" for message in compacted["messages"])
    assert all("tool_call_id" not in message for message in compacted["messages"])
    assert all("tool_calls" not in message for message in compacted["messages"])
    assert "[tool evidence]" in encoded
    assert "[assistant requested an external tool]" in encoded
    assert "Build and verify the Cassandra release safely." in encoded
    assert "Turn this cat into a cute clown." in encoded
    assert "image attachment available to the selected worker" in encoded
    assert "data:image/png;base64" not in encoded
    assert "unbounded test output " * 100 not in encoded
    assert len(encoded) < 18_000


def test_switchyard_classifier_is_bounded_and_fails_open_once():
    """A sick local judge must not hold chat through repeated long retries."""
    config = tomllib.loads(
        _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"]
    )
    classifier = config["llm_clients"]["classifier"]
    assert classifier["base_url"] == "http://127.0.0.1:9008/v1"
    assert classifier["max_retries"] == 0
    for route in ("auto_fast", "auto_balanced"):
        assert config["routes"][route]["recent_turn_window"] == 4
    for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"):
        assert config["routes"][route]["recent_turn_window"] == 6

    deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
    containers = {
        item["name"]: item
        for item in deployment["spec"]["template"]["spec"]["containers"]
    }
    classifier_container = containers["classifier-broker"]
    env = {item["name"]: item["value"] for item in classifier_container["env"]}
    assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60"
    assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier"


def test_image_broker_returns_bytes_and_removes_owner_cache(
    tmp_path: Path, monkeypatch
):
    """The broker must not retain a family user's generated image."""
    broker_path = HERMES / "scripts" / "image_broker.py"
    spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    generated = tmp_path / "generated.png"
    generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image")

    class Provider:
        def generate(self, prompt, aspect, **kwargs):
            assert prompt == "paint a blue sphere"
            assert aspect == "square"
            return {
                "success": True,
                "image": str(generated),
                "model": "gpt-image-2-high",
                "quality": "high",
            }

    monkeypatch.setattr(module, "_PROVIDER", Provider())
    result = module._generate(
        {
            "prompt": "paint a blue sphere",
            "aspect_ratio": "square",
            "model": "gpt-image-2-high",
        }
    )

    assert result["success"] is True
    assert result["image_b64"]
    assert "image" not in result
    assert not generated.exists()


def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes(
    monkeypatch,
):
    """AUTO is hosted-first while explicit local never calls the hosted lane."""
    broker_path = HERMES / "scripts" / "image_broker.py"
    spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    calls = []

    def hosted(payload, model, prompt, aspect):
        calls.append(("hosted", model, prompt, aspect))
        return {"success": False, "error": "hosted refusal"}

    def local(payload, timeout=1800.0):
        calls.append(("local", payload["model"], timeout))
        return {
            "success": True,
            "image_b64": "aW1hZ2U=",
            "model": "flux-2-klein-4b-local",
            "route": "local",
        }

    monkeypatch.setattr(module, "_generate_hosted", hosted)
    monkeypatch.setattr(module, "_local_request", local)
    auto = module._generate(
        {
            "prompt": "colorize this family photograph",
            "aspect_ratio": "portrait",
            "model": "atlas-image-auto-high",
        }
    )
    assert auto["success"] is True
    assert auto["route"] == "local"
    assert auto["hosted_fallback_reason"] == "hosted refusal"
    assert [call[0] for call in calls] == ["hosted", "local"]

    calls.clear()
    explicit = module._generate(
        {
            "prompt": "make a local landscape",
            "aspect_ratio": "landscape",
            "model": "flux-2-klein-4b-local",
        }
    )
    assert explicit["route"] == "local"
    assert [call[0] for call in calls] == ["local"]


def test_image_broker_policy_is_narrow_and_operator_extensible(
    tmp_path: Path, monkeypatch
):
    """Family-photo restoration stays allowed while the hard boundary remains."""
    broker_path = HERMES / "scripts" / "image_broker.py"
    spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    policy = tmp_path / "policy.json"
    policy.write_text('{"additional_blocked_phrases":["site-specific block"]}')
    monkeypatch.setattr(module, "POLICY_PATH", policy)

    assert (
        module._policy_error(
            "Colorize my baby photograph with a lighter natural skin tone"
        )
        is None
    )
    assert "minors" in module._policy_error("Create a sexual image of a child")
    assert module._policy_error("A site-specific block request") == (
        "request is blocked by the operator image policy"
    )


def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
    """FLUX and Wolf share titan-24 while text stays on titan-20."""
    deployment = _documents(HERMES / "local-image-deployment.yaml")[0]
    assert deployment["metadata"]["name"] == "hermes-local-image"
    pod = deployment["spec"]["template"]["spec"]
    assert pod["serviceAccountName"] == "hermes-gpu-runtime"
    local = next(item for item in pod["containers"] if item["name"] == "local-image")
    assert len(pod["containers"]) == 1
    assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1
    assert local["ports"] == [{"name": "local-image", "containerPort": 9004}]
    assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"])
    model_env = {item["name"]: item["value"] for item in local["env"]}
    assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004"
    assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
        "e7b7dc27f91deacad38e78976d1f2b499d76a294"
    )
    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
    assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
        "3221225472"
    )
    assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential"
    assert (
        "nvidia-process-exporter-local.monitoring.svc.cluster.local"
        in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"]
    )
    models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
    assert models_volume["persistentVolumeClaim"]["claimName"] == (
        "hermes-image-models"
    )

    services = _documents(HERMES / "service.yaml")
    image_service = next(
        item for item in services if item["metadata"]["name"] == "hermes-local-image"
    )
    assert image_service["spec"]["selector"] == {"app": "hermes-local-image"}

    handoff_services = _documents(HERMES / "model-gate-deployment.yaml")
    handoff = next(
        item
        for item in handoff_services
        if item["kind"] == "Service"
        and item["metadata"]["name"] == "hermes-gpu-handoff"
    )
    assert handoff["spec"]["ports"][0]["targetPort"] == "handoff"

    ariadne = _documents(
        Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml"
    )[0]
    env = {
        item["name"]: item["value"]
        for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"]
        if "value" in item
    }
    assert env["GAME_MODE_OLLAMA_URL"] == (
        "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434"
    )
    assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local"

    for config_name in (
        "configmap.yaml",
        "agent-configmap.yaml",
        "chat-configmap.yaml",
    ):
        config = _documents(HERMES / config_name)[0]["data"]["config.yaml"]
        assert "gpt-oss:20b" not in config
        assert "atlas-switchyard" in config
    switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][
        "routes.toml"
    ]
    assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard
    assert "qwen2.5:3b-instruct-q4_0" not in switchyard
    assert "route/local/qwen2.5-14b/medium" in switchyard
    assert "Anthropic and Claude name the same provider" in switchyard
    assert "OpenAI and Codex name the same provider" in switchyard
    assert "Choose across every configured Codex and Claude family" in switchyard
    assert "Claude Fable" in switchyard
    assert switchyard.count('Treat "think hard"') == 4
    assert switchyard.count("Never choose below the") >= 5
    switchyard_config = tomllib.loads(switchyard)
    routes = switchyard_config["routes"]
    configured_targets = switchyard_config["targets"]
    for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
        leading_targets = set(routes[route_name]["targets"][:2])
        assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"}
        assert "max_output_tokens" not in routes[route_name]
    for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
        targets = routes[route_name]["targets"]
        selector_targets = routes[route_name]["response_schema"]
        assert not any(target.startswith("local_") for target in targets)
        assert any("fable" in target for target in targets)
        assert "local_qwen" not in selector_targets
        assert "not eligible for foreground" in routes[route_name]["prompt"]
    for route_name in ("auto_deep", "auto_maximum"):
        targets = routes[route_name]["targets"]
        selector_targets = routes[route_name]["response_schema"]
        assert not any(target.endswith("_low") for target in targets)
        assert "_low" not in selector_targets
    maximum_targets = routes["auto_maximum"]["targets"]
    maximum_selector_targets = routes["auto_maximum"]["response_schema"]
    assert not any(target.endswith("_medium") for target in maximum_targets)
    assert "_medium" not in maximum_selector_targets
    assert "absolute high effort floor" in routes["auto_maximum"]["prompt"]
    assert "quality mark was missed" in routes["auto_balanced"]["prompt"]
    assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"]
    assert (
        "Repeated quality misses require xhigh"
        in routes["worker_auto_maximum"]["prompt"]
    )
    assert any(
        target.startswith("local_") for target in routes["manual_local_qwen"]["targets"]
    )
    for route_name in (
        "manual_codex_luna",
        "manual_codex_terra",
        "manual_codex_sol",
        "manual_claude_haiku",
        "manual_claude_fable",
        "manual_claude_sonnet",
        "manual_claude_opus",
    ):
        assert not any(
            target.startswith("local_") for target in routes[route_name]["targets"]
        )
    for provider, families in {
        "codex": ("luna", "terra", "sol"),
        "claude": ("haiku", "fable", "sonnet", "opus"),
    }.items():
        for family in families:
            for effort in ("low", "medium", "high", "xhigh"):
                route = routes[f"manual_{provider}_{family}_{effort}"]
                assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
                assert route["targets"][0] == f"{provider}_{family}_{effort}"
                worker_target = f"worker_{provider}_{family}_{effort}"
                assert worker_target in routes["worker_auto_maximum"]["targets"]
                assert configured_targets[worker_target]["id"] == (
                    f"worker/{provider}/{family}/{effort}"
                )
    for route_name in ("auto_fast", "auto_balanced"):
        prompt = routes[route_name]["prompt"]
        assert "image tool—not the conversational model" in prompt
        assert "Do not select a" in prompt
        assert "local Qwen or Claude target" in prompt
    assert "max_output_tokens" not in routes["worker_auto_maximum"]
    model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
        "model_gate.py"
    ]
    assert "qwen2.5:14b-instruct-q4_0" in model_gate