459 lines
17 KiB
Python
459 lines
17 KiB
Python
"""Agent deployment layout and runtime policy."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
HERMES,
|
|
Path,
|
|
_agent_deployment,
|
|
_services,
|
|
client_config,
|
|
json,
|
|
migration,
|
|
policy,
|
|
pytest,
|
|
yaml,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"command",
|
|
[
|
|
"git push --force origin main",
|
|
"git reset --hard HEAD~1",
|
|
"git clean -fd",
|
|
],
|
|
)
|
|
def test_claude_pretool_hook_blocks_hard_denies(command: str):
|
|
assert policy.denial_reason(command)
|
|
|
|
|
|
def test_claude_pretool_hook_allows_normal_engineering():
|
|
assert policy.denial_reason("pytest -q testing/tests") is None
|
|
assert policy.denial_reason("git push origin feature/hermes") is None
|
|
assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None
|
|
assert policy.denial_reason("flux reconcile kustomization hermes") is None
|
|
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
|
|
|
|
|
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(
|
|
tmp_path: Path,
|
|
):
|
|
state = tmp_path / ".claude.json"
|
|
settings = tmp_path / "settings.json"
|
|
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
|
settings.write_text(
|
|
json.dumps(
|
|
{
|
|
"theme": "dark",
|
|
"permissions": {
|
|
"deny": [
|
|
"Bash(kubectl apply *)",
|
|
"Bash(flux reconcile *)",
|
|
"Bash(vault kv *)",
|
|
"Bash(custom-owner-rule *)",
|
|
]
|
|
},
|
|
}
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
client_config.configure_claude_state(state)
|
|
client_config.configure_claude_settings(settings)
|
|
|
|
state_value = json.loads(state.read_text())
|
|
settings_value = json.loads(settings.read_text())
|
|
assert state_value["promptQueueUseCount"] == 4
|
|
assert state_value["bypassPermissionsModeAccepted"] is True
|
|
assert settings_value["theme"] == "dark"
|
|
assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"]
|
|
assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"]
|
|
assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"]
|
|
assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"]
|
|
assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"]
|
|
hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0]
|
|
assert "claude_command_policy.py" in hook["command"]
|
|
|
|
|
|
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
|
session = tmp_path / "home/.config/herdr/session.json"
|
|
session.parent.mkdir(parents=True)
|
|
session.write_text(
|
|
'{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8"
|
|
)
|
|
binary = tmp_path / "tools/bin/herdr"
|
|
binary.parent.mkdir(parents=True)
|
|
binary.write_text("legacy", encoding="utf-8")
|
|
(tmp_path / "home/.claude").mkdir()
|
|
(tmp_path / "home/.codex").mkdir()
|
|
|
|
archive = migration.archive_legacy_state(tmp_path)
|
|
|
|
value = json.loads(archive.read_text())
|
|
assert value["legacy_session"]["agents"][0]["session_id"] == "abc"
|
|
assert (tmp_path / "home/.claude").is_dir()
|
|
assert (tmp_path / "home/.codex").is_dir()
|
|
assert not binary.exists()
|
|
assert not (tmp_path / "home/.config/herdr").exists()
|
|
|
|
|
|
def test_agent_uses_one_native_kanban_control_plane():
|
|
configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
|
|
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
|
assert config["model"] == {
|
|
"provider": "atlas-switchyard",
|
|
"default": "atlas/auto/balanced",
|
|
"model": "atlas/auto/balanced",
|
|
}
|
|
assert config["kanban"]["dispatch_in_gateway"] is True
|
|
assert config["kanban"]["default_assignee"] == "cli-auto"
|
|
assert config["plugins"]["enabled"] == ["auto-router"]
|
|
|
|
deployment = _agent_deployment()
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
assert pod["enableServiceLinks"] is False
|
|
names = {item["name"] for item in pod["containers"]}
|
|
assert "cli-lane-runner" in names
|
|
assert "terminal" in names
|
|
assert not any("herdr" in name for name in names)
|
|
rendered = (HERMES / "agent-deployment.yaml").read_text()
|
|
assert "herdr server" not in rendered
|
|
assert "herdr-dispatch" not in rendered
|
|
|
|
|
|
def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
|
|
deployment = _agent_deployment()
|
|
containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
lane = containers["cli-lane-runner"]
|
|
environment = {item["name"]: item["value"] for item in lane["env"]}
|
|
|
|
assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
|
|
assert lane["resources"] == {
|
|
"requests": {"cpu": "25m", "memory": "96Mi"},
|
|
"limits": {"cpu": "2", "memory": "6Gi"},
|
|
}
|
|
|
|
|
|
def test_agent_avoids_unhealthy_nodes_and_stays_on_storage_workers():
|
|
"""The owner agent stays off unhealthy and attach-incompatible nodes."""
|
|
pod = _agent_deployment()["spec"]["template"]["spec"]
|
|
terms = pod["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"]
|
|
hostnames = next(
|
|
item
|
|
for item in terms[0]["matchExpressions"]
|
|
if item["key"] == "kubernetes.io/hostname"
|
|
)
|
|
|
|
assert hostnames["operator"] == "NotIn"
|
|
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
|
|
|
# Two OR'd terms: the arm64 pi fleet (fallback) and the amd64 accelerator
|
|
# titan-22 (primary home once the multi-arch image exists).
|
|
assert len(terms) == 2
|
|
pi_fleet = {item["key"]: item for item in terms[0]["matchExpressions"]}
|
|
assert pi_fleet["kubernetes.io/arch"]["values"] == ["arm64"]
|
|
assert pi_fleet["node-role.kubernetes.io/worker"]["values"] == ["true"]
|
|
|
|
titan22 = {item["key"]: item for item in terms[1]["matchExpressions"]}
|
|
assert titan22["kubernetes.io/arch"]["values"] == ["amd64"]
|
|
assert titan22["kubernetes.io/hostname"]["values"] == ["titan-22"]
|
|
assert titan22["node-role.kubernetes.io/accelerator"]["values"] == ["true"]
|
|
# titan-22 access must NOT depend on the generic worker label.
|
|
assert "node-role.kubernetes.io/worker" not in titan22
|
|
|
|
preferences = pod["affinity"]["nodeAffinity"][
|
|
"preferredDuringSchedulingIgnoredDuringExecution"
|
|
]
|
|
# titan-22 is the STRONG primary (max weight 100) and outranks the pi-fleet
|
|
# nudge (50), so hermes-agent actually lives on titan-22 when schedulable.
|
|
assert [item["weight"] for item in preferences] == [100, 50]
|
|
primary = preferences[0]["preference"]["matchExpressions"][0]
|
|
assert primary["key"] == "kubernetes.io/hostname"
|
|
assert primary["values"] == ["titan-22"]
|
|
|
|
# The pod tolerates the accelerator hard taint so it can consider titan-22.
|
|
assert {
|
|
"key": "node-role.kubernetes.io/accelerator",
|
|
"operator": "Equal",
|
|
"value": "true",
|
|
"effect": "NoSchedule",
|
|
} in pod["tolerations"]
|
|
# It also tolerates the soft media-primary taint that titan-22 currently
|
|
# carries, so the weight-100 titan-22 preference is not offset by the
|
|
# PreferNoSchedule penalty and placement there is deterministic.
|
|
assert {
|
|
"key": "atlas.bstein.dev/media-primary",
|
|
"operator": "Equal",
|
|
"value": "true",
|
|
"effect": "PreferNoSchedule",
|
|
} in pod["tolerations"]
|
|
|
|
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
|
assert hermes["resources"]["requests"] == {
|
|
"cpu": "125m",
|
|
"memory": "320Mi",
|
|
}
|
|
|
|
node_labels = (
|
|
Path(__file__).parents[2]
|
|
/ "infrastructure/core/node-prefer-noschedule-cronjob.yaml"
|
|
).read_text()
|
|
assert "k label node titan-21 longhorn-host-" in node_labels
|
|
|
|
|
|
def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
|
deployment = _agent_deployment()
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
containers = {item["name"]: item for item in pod["containers"]}
|
|
assert "webui" not in containers
|
|
|
|
assert "dashboard" not in containers
|
|
hermes = containers["hermes"]
|
|
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
|
assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600"
|
|
assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600"
|
|
assert hermes["command"] == ["/bin/sh", "-ec"]
|
|
startup = hermes["args"][0]
|
|
assert ". /opt/data/.env" in startup
|
|
assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup
|
|
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
|
assert hermes_env["HERMES_DASHBOARD"] == "1"
|
|
assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1"
|
|
assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119"
|
|
assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
|
assert hermes["securityContext"]["runAsUser"] == 0
|
|
assert hermes["securityContext"]["runAsGroup"] == 0
|
|
for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"):
|
|
probe = hermes[probe_name]
|
|
assert probe["exec"]["command"] == [
|
|
"curl",
|
|
"-fsS",
|
|
"http://127.0.0.1:9119/api/status",
|
|
]
|
|
|
|
terminal = containers["terminal"]
|
|
command = terminal["args"][0]
|
|
assert "--base-path /terminal" in command
|
|
assert "--check-origin" not in command
|
|
assert "/usr/bin/tmux new-session -A" in command
|
|
assert "--continue" in command
|
|
assert "--yolo" in command
|
|
terminal_env = {item["name"]: item["value"] for item in terminal["env"]}
|
|
assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
|
|
|
claude_broker = containers["claude-broker"]
|
|
claude_env = {item["name"]: item["value"] for item in claude_broker["env"]}
|
|
assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2"
|
|
assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
|
assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
|
|
|
args = containers["oauth2-proxy"]["args"]
|
|
terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/"
|
|
dashboard_upstream = "--upstream=http://127.0.0.1:9119/"
|
|
assert terminal_upstream in args
|
|
assert dashboard_upstream in args
|
|
assert args.index(terminal_upstream) < args.index(dashboard_upstream)
|
|
assert "--pass-host-header=false" in args
|
|
assert "--cookie-refresh=19m" in args
|
|
assert "--session-store-type=redis" in args
|
|
assert any(
|
|
arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
|
|
for arg in args
|
|
)
|
|
|
|
patch_init = next(
|
|
item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway"
|
|
)
|
|
assert patch_init["command"][-1] == "/patched/server.py"
|
|
for name in ("hermes", "terminal"):
|
|
mounts = containers[name]["volumeMounts"]
|
|
assert {
|
|
"name": "tui-gateway-patch",
|
|
"mountPath": "/opt/hermes/tui_gateway/server.py",
|
|
"subPath": "server.py",
|
|
} in mounts
|
|
|
|
ingress_documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text())
|
|
if item
|
|
]
|
|
middlewares = {
|
|
item["metadata"]["name"]: item
|
|
for item in ingress_documents
|
|
if item["kind"] == "Middleware"
|
|
}
|
|
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
|
"replacement"
|
|
].endswith("/terminal/")
|
|
assert (
|
|
middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
|
"customRequestHeaders"
|
|
]["Origin"]
|
|
== "http://127.0.0.1:9119"
|
|
)
|
|
ingresses = {
|
|
item["metadata"]["name"]: item
|
|
for item in ingress_documents
|
|
if item["kind"] == "Ingress"
|
|
}
|
|
assert (
|
|
ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
|
]
|
|
== "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
|
)
|
|
assert (
|
|
ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
|
]
|
|
== "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
|
)
|
|
|
|
|
|
def test_broker_services_survive_sibling_container_readiness_loss():
|
|
services = _services()
|
|
|
|
for name in (
|
|
"hermes-image-broker",
|
|
"hermes-codex-broker",
|
|
"hermes-local-image",
|
|
"hermes-claude-broker",
|
|
):
|
|
assert services[name]["spec"]["publishNotReadyAddresses"] is True
|
|
|
|
# The agent web endpoints route through this Service to the same pod the
|
|
# cli-lane-runner readiness probe gates; a deferred lane image must not
|
|
# take down the dashboard or terminal.
|
|
agent_auth = next(
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text())
|
|
if item
|
|
and item["kind"] == "Service"
|
|
and item["metadata"]["name"] == "oauth2-proxy-hermes-agent"
|
|
)
|
|
assert agent_auth["spec"]["selector"] == {"app": "hermes-agent"}
|
|
assert agent_auth["spec"]["publishNotReadyAddresses"] is True
|
|
|
|
|
|
def test_cli_lane_domain_modules_are_all_mounted_with_the_runner():
|
|
manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
|
|
coordinator = next(
|
|
item
|
|
for item in manifest["configMapGenerator"]
|
|
if item["name"] == "hermes-coordinator"
|
|
)
|
|
mounted = {entry.split("=", 1)[0] for entry in coordinator["files"]}
|
|
expected = {
|
|
f"cli_lane_{domain}.py"
|
|
for domain in (
|
|
"board",
|
|
"config",
|
|
"dispatch",
|
|
"evidence",
|
|
"execution",
|
|
"files",
|
|
"finalization",
|
|
"goal",
|
|
"prompt",
|
|
"provider",
|
|
"quarantine",
|
|
"records",
|
|
"recovery",
|
|
"retention",
|
|
"routing",
|
|
"runner",
|
|
)
|
|
}
|
|
assert expected <= mounted
|
|
|
|
|
|
def test_cli_lane_config_refresh_does_not_restart_active_work():
|
|
"""Keep projected script refreshes separate from pod lifecycle changes."""
|
|
manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
|
|
coordinator = next(
|
|
item
|
|
for item in manifest["configMapGenerator"]
|
|
if item["name"] == "hermes-coordinator"
|
|
)
|
|
deployment = _agent_deployment()
|
|
lane = next(
|
|
item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
if item["name"] == "cli-lane-runner"
|
|
)
|
|
mount = next(item for item in lane["volumeMounts"] if item["name"] == "coordinator")
|
|
|
|
assert coordinator["options"]["disableNameSuffixHash"] is True
|
|
assert mount == {
|
|
"name": "coordinator",
|
|
"mountPath": "/opt/coordinator",
|
|
"readOnly": True,
|
|
}
|
|
assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get(
|
|
"metadata", {}
|
|
).get("annotations", {})
|
|
assert "startupProbe" not in lane
|
|
assert "livenessProbe" not in lane
|
|
assert lane["readinessProbe"] == {
|
|
"httpGet": {"path": "/healthz", "port": "lane-metrics"},
|
|
"initialDelaySeconds": 2,
|
|
"periodSeconds": 5,
|
|
"timeoutSeconds": 5,
|
|
"failureThreshold": 3,
|
|
}
|
|
environment = {item["name"]: item["value"] for item in lane["env"]}
|
|
assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60"
|
|
|
|
|
|
def test_agent_dashboard_reconnects_all_transient_websockets():
|
|
dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "eventsRetryAttempt.current" in dockerfile
|
|
assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
|
|
assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
|
|
assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile
|
|
assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile
|
|
assert dockerfile.count("' await api.getSessions(1, 0,") == 2
|
|
assert ".then(() => gw.connect())" in dockerfile
|
|
assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile
|
|
assert "dashboard token rotated by a server restart" in dockerfile
|
|
|
|
|
|
def test_agent_image_runs_execution_safety_patch_and_regressions():
|
|
dockerfiles = HERMES.parents[1] / "dockerfiles"
|
|
dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8")
|
|
dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
for name in (
|
|
"patch-hermes-execution-safety.py",
|
|
"hermes-execution-safety-regression.py",
|
|
"hermes_execution_patch_support.py",
|
|
"patch_hermes_run_safety.py",
|
|
"patch_hermes_decomposition_safety.py",
|
|
"hermes_execution_regression_support.py",
|
|
"hermes_run_safety_regression.py",
|
|
"hermes_decomposition_safety_regression.py",
|
|
):
|
|
assert f"COPY dockerfiles/{name}" in dockerfile
|
|
assert f"!dockerfiles/{name}" in dockerignore
|
|
assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in (
|
|
dockerfile
|
|
)
|
|
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
|
|
dockerfile
|
|
)
|
|
assert "COPY services/hermes/scripts/*.py /tmp/hermes-lane-regression/" in dockerfile
|
|
assert "!services/hermes/scripts/*.py" in dockerignore
|
|
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
|