atlas-iac/testing/tests/test_hermes_agent_layout.py

"""Agent deployment layout and runtime policy."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    HERMES,
    Path,
    _agent_deployment,
    _services,
    client_config,
    json,
    migration,
    policy,
    pytest,
    yaml,
)


@pytest.mark.parametrize(
    "command",
    [
        "git push --force origin main",
        "git reset --hard HEAD~1",
        "git clean -fd",
    ],
)
def test_claude_pretool_hook_blocks_hard_denies(command: str):
    assert policy.denial_reason(command)


def test_claude_pretool_hook_allows_normal_engineering():
    assert policy.denial_reason("pytest -q testing/tests") is None
    assert policy.denial_reason("git push origin feature/hermes") is None
    assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None
    assert policy.denial_reason("flux reconcile kustomization hermes") is None
    assert policy.denial_reason("vault kv get kv/atlas/hermes") is None


def test_claude_settings_preserve_state_and_install_three_guardrail_layers(
    tmp_path: Path,
):
    state = tmp_path / ".claude.json"
    settings = tmp_path / "settings.json"
    state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
    settings.write_text(
        json.dumps(
            {
                "theme": "dark",
                "permissions": {
                    "deny": [
                        "Bash(kubectl apply *)",
                        "Bash(flux reconcile *)",
                        "Bash(vault kv *)",
                        "Bash(custom-owner-rule *)",
                    ]
                },
            }
        )
        + "\n",
        encoding="utf-8",
    )

    client_config.configure_claude_state(state)
    client_config.configure_claude_settings(settings)

    state_value = json.loads(state.read_text())
    settings_value = json.loads(settings.read_text())
    assert state_value["promptQueueUseCount"] == 4
    assert state_value["bypassPermissionsModeAccepted"] is True
    assert settings_value["theme"] == "dark"
    assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"]
    assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"]
    assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"]
    assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"]
    assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"]
    hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0]
    assert "claude_command_policy.py" in hook["command"]


def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
    session = tmp_path / "home/.config/herdr/session.json"
    session.parent.mkdir(parents=True)
    session.write_text(
        '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8"
    )
    binary = tmp_path / "tools/bin/herdr"
    binary.parent.mkdir(parents=True)
    binary.write_text("legacy", encoding="utf-8")
    (tmp_path / "home/.claude").mkdir()
    (tmp_path / "home/.codex").mkdir()

    archive = migration.archive_legacy_state(tmp_path)

    value = json.loads(archive.read_text())
    assert value["legacy_session"]["agents"][0]["session_id"] == "abc"
    assert (tmp_path / "home/.claude").is_dir()
    assert (tmp_path / "home/.codex").is_dir()
    assert not binary.exists()
    assert not (tmp_path / "home/.config/herdr").exists()


def test_agent_uses_one_native_kanban_control_plane():
    configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
    config = yaml.safe_load(configmap["data"]["config.yaml"])
    assert config["model"] == {
        "provider": "atlas-switchyard",
        "default": "atlas/auto/balanced",
        "model": "atlas/auto/balanced",
    }
    assert config["kanban"]["dispatch_in_gateway"] is True
    assert config["kanban"]["default_assignee"] == "cli-auto"
    assert config["plugins"]["enabled"] == ["auto-router"]

    deployment = _agent_deployment()
    pod = deployment["spec"]["template"]["spec"]
    assert pod["enableServiceLinks"] is False
    names = {item["name"] for item in pod["containers"]}
    assert "cli-lane-runner" in names
    assert "terminal" in names
    assert not any("herdr" in name for name in names)
    rendered = (HERMES / "agent-deployment.yaml").read_text()
    assert "herdr server" not in rendered
    assert "herdr-dispatch" not in rendered


def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
    deployment = _agent_deployment()
    containers = {
        item["name"]: item
        for item in deployment["spec"]["template"]["spec"]["containers"]
    }
    lane = containers["cli-lane-runner"]
    environment = {item["name"]: item["value"] for item in lane["env"]}

    assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
    assert lane["resources"] == {
        "requests": {"cpu": "25m", "memory": "96Mi"},
        "limits": {"cpu": "2", "memory": "6Gi"},
    }


def test_agent_avoids_unhealthy_nodes_and_stays_on_storage_workers():
    """The owner agent stays off unhealthy and attach-incompatible nodes."""
    pod = _agent_deployment()["spec"]["template"]["spec"]
    terms = pod["affinity"]["nodeAffinity"][
        "requiredDuringSchedulingIgnoredDuringExecution"
    ]["nodeSelectorTerms"]
    hostnames = next(
        item
        for item in terms[0]["matchExpressions"]
        if item["key"] == "kubernetes.io/hostname"
    )

    assert hostnames["operator"] == "NotIn"
    assert set(hostnames["values"]) >= {"titan-04", "titan-19"}

    # Multi-arch: the arm64 fleet term is preserved as a fallback (OR'd), plus a
    # second term that permits ONLY amd64 titan-22 as spare capacity.
    assert len(terms) == 2
    arch_values = {
        expr["values"][0]
        for term in terms
        for expr in term["matchExpressions"]
        if expr["key"] == "kubernetes.io/arch"
    }
    assert arch_values == {"arm64", "amd64"}
    titan22_term = next(
        term
        for term in terms
        if any(
            expr["key"] == "kubernetes.io/hostname" and expr["operator"] == "In"
            for expr in term["matchExpressions"]
        )
    )
    titan22_hosts = next(
        expr
        for expr in titan22_term["matchExpressions"]
        if expr["key"] == "kubernetes.io/hostname"
    )
    assert titan22_hosts["values"] == ["titan-22"]

    preferences = pod["affinity"]["nodeAffinity"][
        "preferredDuringSchedulingIgnoredDuringExecution"
    ]
    # rpi5 nudge preserved; titan-22 nudge added at equal weight (not forced).
    assert [item["weight"] for item in preferences] == [100, 100]

    # hermes-agent may use titan-22 despite the media-primary taint, without
    # changing jellyfin's own scheduling.
    tolerations = pod.get("tolerations", [])
    assert {
        "key": "atlas.bstein.dev/media-primary",
        "operator": "Equal",
        "value": "true",
        "effect": "PreferNoSchedule",
    } in tolerations

    hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
    assert hermes["resources"]["requests"] == {
        "cpu": "125m",
        "memory": "320Mi",
    }

    node_labels = (
        Path(__file__).parents[2]
        / "infrastructure/core/node-prefer-noschedule-cronjob.yaml"
    ).read_text()
    assert "k label node titan-21 longhorn-host-" in node_labels


def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
    deployment = _agent_deployment()
    pod = deployment["spec"]["template"]["spec"]
    containers = {item["name"]: item for item in pod["containers"]}
    assert "webui" not in containers

    assert "dashboard" not in containers
    hermes = containers["hermes"]
    hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
    assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600"
    assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600"
    assert hermes["command"] == ["/bin/sh", "-ec"]
    startup = hermes["args"][0]
    assert ". /opt/data/.env" in startup
    assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup
    hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
    assert hermes_env["HERMES_DASHBOARD"] == "1"
    assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1"
    assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119"
    assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
    assert hermes["securityContext"]["runAsUser"] == 0
    assert hermes["securityContext"]["runAsGroup"] == 0
    for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"):
        probe = hermes[probe_name]
        assert probe["exec"]["command"] == [
            "curl",
            "-fsS",
            "http://127.0.0.1:9119/api/status",
        ]

    terminal = containers["terminal"]
    command = terminal["args"][0]
    assert "--base-path /terminal" in command
    assert "--check-origin" not in command
    assert "/usr/bin/tmux new-session -A" in command
    assert "--continue" in command
    assert "--yolo" in command
    terminal_env = {item["name"]: item["value"] for item in terminal["env"]}
    assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"

    claude_broker = containers["claude-broker"]
    claude_env = {item["name"]: item["value"] for item in claude_broker["env"]}
    assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2"
    assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"}
    assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"}

    args = containers["oauth2-proxy"]["args"]
    terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/"
    dashboard_upstream = "--upstream=http://127.0.0.1:9119/"
    assert terminal_upstream in args
    assert dashboard_upstream in args
    assert args.index(terminal_upstream) < args.index(dashboard_upstream)
    assert "--pass-host-header=false" in args
    assert "--cookie-refresh=19m" in args
    assert "--session-store-type=redis" in args
    assert any(
        arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
        for arg in args
    )

    patch_init = next(
        item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway"
    )
    assert patch_init["command"][-1] == "/patched/server.py"
    for name in ("hermes", "terminal"):
        mounts = containers[name]["volumeMounts"]
        assert {
            "name": "tui-gateway-patch",
            "mountPath": "/opt/hermes/tui_gateway/server.py",
            "subPath": "server.py",
        } in mounts

    ingress_documents = [
        item
        for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text())
        if item
    ]
    middlewares = {
        item["metadata"]["name"]: item
        for item in ingress_documents
        if item["kind"] == "Middleware"
    }
    assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
        "replacement"
    ].endswith("/terminal/")
    assert (
        middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
            "customRequestHeaders"
        ]["Origin"]
        == "http://127.0.0.1:9119"
    )
    ingresses = {
        item["metadata"]["name"]: item
        for item in ingress_documents
        if item["kind"] == "Ingress"
    }
    assert (
        ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
            "traefik.ingress.kubernetes.io/router.middlewares"
        ]
        == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
    )
    assert (
        ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
            "traefik.ingress.kubernetes.io/router.middlewares"
        ]
        == "hermes-hermes-agent-terminal-slash@kubernetescrd"
    )


def test_broker_services_survive_sibling_container_readiness_loss():
    services = _services()

    for name in (
        "hermes-image-broker",
        "hermes-codex-broker",
        "hermes-local-image",
        "hermes-claude-broker",
    ):
        assert services[name]["spec"]["publishNotReadyAddresses"] is True

    # The agent web endpoints route through this Service to the same pod the
    # cli-lane-runner readiness probe gates; a deferred lane image must not
    # take down the dashboard or terminal.
    agent_auth = next(
        item
        for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text())
        if item
        and item["kind"] == "Service"
        and item["metadata"]["name"] == "oauth2-proxy-hermes-agent"
    )
    assert agent_auth["spec"]["selector"] == {"app": "hermes-agent"}
    assert agent_auth["spec"]["publishNotReadyAddresses"] is True


def test_cli_lane_domain_modules_are_all_mounted_with_the_runner():
    manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
    coordinator = next(
        item
        for item in manifest["configMapGenerator"]
        if item["name"] == "hermes-coordinator"
    )
    mounted = {entry.split("=", 1)[0] for entry in coordinator["files"]}
    expected = {
        f"cli_lane_{domain}.py"
        for domain in (
            "board",
            "config",
            "dispatch",
            "evidence",
            "execution",
            "files",
            "finalization",
            "goal",
            "prompt",
            "provider",
            "quarantine",
            "records",
            "recovery",
            "retention",
            "routing",
            "runner",
        )
    }
    assert expected <= mounted


def test_cli_lane_config_refresh_does_not_restart_active_work():
    """Keep projected script refreshes separate from pod lifecycle changes."""
    manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
    coordinator = next(
        item
        for item in manifest["configMapGenerator"]
        if item["name"] == "hermes-coordinator"
    )
    deployment = _agent_deployment()
    lane = next(
        item
        for item in deployment["spec"]["template"]["spec"]["containers"]
        if item["name"] == "cli-lane-runner"
    )
    mount = next(item for item in lane["volumeMounts"] if item["name"] == "coordinator")

    assert coordinator["options"]["disableNameSuffixHash"] is True
    assert mount == {
        "name": "coordinator",
        "mountPath": "/opt/coordinator",
        "readOnly": True,
    }
    assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get(
        "metadata", {}
    ).get("annotations", {})
    assert "startupProbe" not in lane
    assert "livenessProbe" not in lane
    assert lane["readinessProbe"] == {
        "exec": {
            "command": [
                "/opt/hermes/.venv/bin/python",
                "/opt/coordinator/cli_lane_capabilities.py",
            ]
        },
        "initialDelaySeconds": 2,
        "periodSeconds": 5,
        "timeoutSeconds": 5,
        "failureThreshold": 3,
    }
    environment = {item["name"]: item["value"] for item in lane["env"]}
    assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60"


def test_agent_dashboard_reconnects_all_transient_websockets():
    dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text(
        encoding="utf-8"
    )
    assert "eventsRetryAttempt.current" in dockerfile
    assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
    assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
    assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile
    assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile
    assert dockerfile.count("'        await api.getSessions(1, 0,") == 2
    assert ".then(() => gw.connect())" in dockerfile
    assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile
    assert "dashboard token rotated by a server restart" in dockerfile


def test_agent_image_runs_execution_safety_patch_and_regressions():
    dockerfiles = HERMES.parents[1] / "dockerfiles"
    dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8")
    dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text(
        encoding="utf-8"
    )

    for name in (
        "patch-hermes-execution-safety.py",
        "hermes-execution-safety-regression.py",
        "hermes_execution_patch_support.py",
        "patch_hermes_run_safety.py",
        "patch_hermes_decomposition_safety.py",
        "hermes_execution_regression_support.py",
        "hermes_run_safety_regression.py",
        "hermes_decomposition_safety_regression.py",
    ):
        assert f"COPY dockerfiles/{name}" in dockerfile
        assert f"!dockerfiles/{name}" in dockerignore
    assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in (
        dockerfile
    )
    assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
        dockerfile
    )
    assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
    assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
    assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile