750dfa241f
The sticky-block gate added in the previous commit classifies a task from the `created` event payload that upstream `create_task` writes. That producer is code we do not own, so trusting it silently was the gap: if upstream renamed the key, dropped it, or stopped deriving it from `initial_status`, the image would still build and ship a consumer that mis-classifies every task it reads. Anchor the producer contract at build time, before the regression suite runs, with three assert-only preconditions: the `initial_status="blocked"` park resolves `task_status` to `"blocked"`, every non-park creation resolves it to something else, and the `created` event carries that same variable under `"status"`. None of them rewrite the producer. Textual anchors cannot see dataflow, so add the runtime net the reviewer asked for. The suite now drives the real API: create + claim an ordinary task, trip the circuit breaker once at failure_limit=1 so it parks with a `gave_up` event (leaving its own `created` event as the most recent create/block/unblock row), then recompute at failure_limit=2 and require promotion to ready. That case is red under an unconditional-true created predicate and red under producer drift that labels every created event blocked, while the explicit block/unblock, dependency-promotion and circuit-breaker-at-current-limit cases stay green. Non-blocked and malformed created payloads are pinned as controls, and the gate now rejects non-dict payloads rather than trusting `.get`. Also make the live placement correction durable: titan-04 is cordoned after repeated kernel undervoltage and kubelet failure and titan-19 was probe/Longhorn unstable under worker load, so both join the hard NotIn list; titan-05 is healthy but sits at 3592m/3600m requested CPU, so the main hermes container gives back 50m (350m -> 300m) to schedule there. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1972 lines
69 KiB
Python
1972 lines
69 KiB
Python
"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import signal
|
|
import sys
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
HERMES = Path(__file__).parents[2] / "services/hermes"
|
|
KEYCLOAK = Path(__file__).parents[2] / "services/keycloak"
|
|
FLUX_HERMES = (
|
|
Path(__file__).parents[2]
|
|
/ "clusters/atlas/flux-system/applications/hermes/kustomization.yaml"
|
|
)
|
|
|
|
|
|
def _load(name: str):
|
|
spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
lanes = _load("cli_lane_runner")
|
|
policy = _load("claude_command_policy")
|
|
migration = _load("migrate_herdr_state")
|
|
auth_patch = _load("patch_hermes_auth")
|
|
tui_gateway_patch = _load("patch_tui_gateway")
|
|
codex_runtime_patch = _load("patch_codex_runtime")
|
|
ttyd_patch = _load("patch_ttyd_index")
|
|
client_config = _load("configure_agent_clients")
|
|
|
|
|
|
def _agent_deployment() -> dict:
|
|
return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
|
|
|
|
|
|
def _services() -> dict[str, dict]:
|
|
return {
|
|
item["metadata"]["name"]: item
|
|
for item in yaml.safe_load_all((HERMES / "service.yaml").read_text())
|
|
if item
|
|
}
|
|
|
|
|
|
def _oauth_deployment(name: str) -> dict:
|
|
documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text())
|
|
if item
|
|
]
|
|
return next(
|
|
item
|
|
for item in documents
|
|
if item["kind"] == "Deployment" and item["metadata"]["name"] == name
|
|
)
|
|
|
|
|
|
class _SwitchyardResponse:
|
|
"""Minimal context-managed response used by routing contract tests."""
|
|
|
|
def __init__(self, selected: str, rationale: str = "local classifier vote"):
|
|
self.headers = {
|
|
"x-model-router-selected-model": selected,
|
|
"x-model-router-rationale": rationale,
|
|
}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self):
|
|
return b"{}"
|
|
|
|
|
|
def test_auto_lane_uses_switchyard_worker_decision():
|
|
observed = {}
|
|
|
|
def route(request, timeout):
|
|
observed["payload"] = json.loads(request.data)
|
|
observed["timeout"] = timeout
|
|
return _SwitchyardResponse("worker/claude/claude-opus-5/xhigh")
|
|
|
|
route = lanes.select_route(
|
|
"Perform the security review.",
|
|
"cli-auto",
|
|
open_request=route,
|
|
)
|
|
|
|
assert route.provider == "claude"
|
|
assert route.model == "claude-opus-5"
|
|
assert route.effort == "xhigh"
|
|
assert route.classifier == "switchyard-classifier"
|
|
assert observed["payload"]["model"] == "atlas/worker/auto/maximum"
|
|
assert observed["timeout"] == 60
|
|
|
|
|
|
def test_manual_lane_is_still_enforced_by_switchyard():
|
|
observed = {}
|
|
|
|
def route(request, timeout):
|
|
observed["payload"] = json.loads(request.data)
|
|
return _SwitchyardResponse(
|
|
"worker/claude/claude-sonnet-5/high", "manual route"
|
|
)
|
|
|
|
route = lanes.select_route(
|
|
"Implement it.",
|
|
"cli-claude-high",
|
|
open_request=route,
|
|
)
|
|
|
|
assert route.provider == "claude"
|
|
assert route.effort == "high"
|
|
assert route.classifier == "switchyard-manual"
|
|
assert route.reason == "manual route"
|
|
assert observed["payload"]["model"] == "atlas/worker/manual/claude/high"
|
|
|
|
|
|
def test_cross_provider_retry_passes_failed_provider_to_switchyard():
|
|
observed = {}
|
|
|
|
def route(request, timeout):
|
|
observed["payload"] = json.loads(request.data)
|
|
return _SwitchyardResponse("worker/claude/claude-sonnet-5/high")
|
|
|
|
route = lanes.select_route(
|
|
"Retry after capacity exhaustion.",
|
|
"cli-auto",
|
|
exclude_provider="codex",
|
|
open_request=route,
|
|
)
|
|
|
|
assert route.provider == "claude"
|
|
assert route.classifier == "switchyard-classifier"
|
|
content = observed["payload"]["messages"][0]["content"]
|
|
assert "codex provider failed or exhausted capacity" in content
|
|
|
|
|
|
def test_classifier_cannot_select_a_freshly_excluded_provider():
|
|
payloads = []
|
|
|
|
def route(request, timeout):
|
|
payload = json.loads(request.data)
|
|
payloads.append(payload)
|
|
if len(payloads) == 1:
|
|
return _SwitchyardResponse("worker/claude/opus/xhigh")
|
|
return _SwitchyardResponse("worker/codex/sol/xhigh", "healthy route")
|
|
|
|
selected = lanes.select_route(
|
|
"Perform a consequential review.",
|
|
"cli-auto",
|
|
exclude_provider="claude",
|
|
exclude_reason="is unavailable according to fresh native health",
|
|
open_request=route,
|
|
)
|
|
|
|
assert selected.provider == "codex"
|
|
assert selected.effort == "xhigh"
|
|
assert selected.classifier == "switchyard-classifier-health-guard"
|
|
assert payloads[0]["model"] == "atlas/worker/auto/maximum"
|
|
assert payloads[1]["model"] == "atlas/worker/manual/codex/xhigh"
|
|
assert "claude provider is unavailable" in payloads[0]["messages"][0]["content"]
|
|
|
|
|
|
def test_fresh_native_health_excludes_only_one_proven_down_provider(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
paths = {
|
|
"codex": tmp_path / "codex.json",
|
|
"claude": tmp_path / "claude.json",
|
|
}
|
|
paths["codex"].write_text('{"state":"available"}\n', encoding="utf-8")
|
|
paths["claude"].write_text('{"state":"unavailable"}\n', encoding="utf-8")
|
|
monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
|
|
|
|
now = max(path.stat().st_mtime for path in paths.values())
|
|
assert lanes.fresh_unavailable_provider(now=now) == "claude"
|
|
|
|
paths["codex"].write_text('{"state":"unavailable"}\n', encoding="utf-8")
|
|
now = max(path.stat().st_mtime for path in paths.values())
|
|
assert lanes.fresh_unavailable_provider(now=now) is None
|
|
|
|
|
|
def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch):
|
|
paths = {
|
|
"codex": tmp_path / "codex.json",
|
|
"claude": tmp_path / "claude.json",
|
|
}
|
|
paths["codex"].write_text(
|
|
'{"state":"available","authenticated":true}\n', encoding="utf-8"
|
|
)
|
|
paths["claude"].write_text(
|
|
'{"state":"unavailable","authenticated":false}\n', encoding="utf-8"
|
|
)
|
|
monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
|
|
|
|
stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60
|
|
assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude"
|
|
|
|
paths["claude"].write_text(
|
|
'{"state":"unavailable","authenticated":true}\n', encoding="utf-8"
|
|
)
|
|
transient_stale = paths["claude"].stat().st_mtime + 10 * 60
|
|
assert lanes.fresh_unavailable_provider(now=transient_stale) is None
|
|
|
|
|
|
def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch):
|
|
"""Kanban workers must not fall back to credential-free persistent homes."""
|
|
monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex")
|
|
monkeypatch.setenv("CLAUDE_CONFIG_DIR", "/runtime-access/claude")
|
|
|
|
env = lanes._base_env()
|
|
|
|
assert env["HOME"] == str(lanes.DATA_ROOT / "home")
|
|
assert env["CODEX_HOME"] == "/runtime-access/codex"
|
|
assert env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
|
|
assert env["GIT_TERMINAL_PROMPT"] == "0"
|
|
|
|
|
|
def test_claude_session_is_reserved_before_first_process(tmp_path: Path, monkeypatch):
|
|
state: dict = {}
|
|
state_file = tmp_path / "state.json"
|
|
observed = {}
|
|
|
|
def fake_stream(command, **kwargs):
|
|
observed["command"] = command
|
|
observed["state"] = json.loads(state_file.read_text())
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
|
|
|
lanes.run_provider(
|
|
route,
|
|
"Do the work.",
|
|
tmp_path,
|
|
state,
|
|
state_file,
|
|
tmp_path / "worker.log",
|
|
lambda _note: True,
|
|
60,
|
|
)
|
|
|
|
reserved = observed["state"]["claude_session_id"]
|
|
assert reserved
|
|
assert ["--session-id", reserved] == observed["command"][-4:-2]
|
|
|
|
|
|
def test_claude_recovery_uses_verified_error_state_machine(tmp_path: Path, monkeypatch):
|
|
state = {"claude_session_id": "d4f91f59-88cb-43d0-a64f-f3284688fe9e", "claude_started": True}
|
|
state_file = tmp_path / "state.json"
|
|
calls = []
|
|
|
|
def fake_stream(command, **kwargs):
|
|
calls.append(command)
|
|
if len(calls) == 1:
|
|
return lanes.ProcessResult(
|
|
1,
|
|
"No conversation found with session ID: d4f91f59-88cb-43d0-a64f-f3284688fe9e",
|
|
None,
|
|
False,
|
|
)
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
|
lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
|
|
|
|
assert "--resume" in calls[0]
|
|
assert "--session-id" in calls[1]
|
|
assert calls[0][calls[0].index("--resume") + 1] == calls[1][calls[1].index("--session-id") + 1]
|
|
|
|
|
|
def test_claude_collision_never_double_starts_a_reserved_session(tmp_path: Path, monkeypatch):
|
|
state = {"claude_session_id": "24927cee-f3da-4bcf-b624-602c22860155"}
|
|
state_file = tmp_path / "state.json"
|
|
calls = []
|
|
|
|
def fake_stream(command, **kwargs):
|
|
calls.append(command)
|
|
if len(calls) == 1:
|
|
return lanes.ProcessResult(1, "Session ID already in use", None, False)
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
|
lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
|
|
|
|
assert "--session-id" in calls[0]
|
|
assert "--resume" in calls[1]
|
|
assert calls[0][calls[0].index("--session-id") + 1] == calls[1][calls[1].index("--resume") + 1]
|
|
|
|
|
|
def test_codex_thread_started_is_persisted_before_completion(tmp_path: Path):
|
|
state: dict = {}
|
|
state_file = tmp_path / "state.json"
|
|
|
|
lanes._event_payload(
|
|
"codex",
|
|
json.dumps({"type": "thread.started", "thread_id": "thread-123"}),
|
|
state,
|
|
state_file,
|
|
)
|
|
|
|
assert json.loads(state_file.read_text())["codex_thread_id"] == "thread-123"
|
|
|
|
|
|
def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch):
|
|
state = {"codex_thread_id": "failed-thread"}
|
|
state_file = tmp_path / "state.json"
|
|
calls = []
|
|
|
|
def fake_stream(command, **kwargs):
|
|
calls.append(command)
|
|
if len(calls) == 1:
|
|
return lanes.ProcessResult(
|
|
1,
|
|
"thread/resume failed: no rollout found for thread id failed-thread",
|
|
None,
|
|
False,
|
|
)
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
|
route = lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
)
|
|
|
|
result = lanes.run_provider(
|
|
route,
|
|
"Continue.",
|
|
tmp_path,
|
|
state,
|
|
state_file,
|
|
tmp_path / "log",
|
|
lambda _: True,
|
|
60,
|
|
)
|
|
|
|
assert result.returncode == 0
|
|
assert "resume" in calls[0]
|
|
assert "resume" not in calls[1]
|
|
assert "codex_thread_id" not in json.loads(state_file.read_text())
|
|
|
|
|
|
def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path):
|
|
result = lanes.stream_process(
|
|
[sys.executable, "-c", "print('authentication work completed')"],
|
|
provider="codex",
|
|
cwd=tmp_path,
|
|
env=dict(os.environ),
|
|
log_path=tmp_path / "worker.log",
|
|
state={},
|
|
state_file=tmp_path / "state.json",
|
|
heartbeat=lambda _note: True,
|
|
max_runtime=60,
|
|
)
|
|
|
|
assert result.returncode == 0
|
|
assert result.capacity_failure is False
|
|
|
|
|
|
def test_worker_process_uses_isolated_process_session(monkeypatch, tmp_path: Path):
|
|
original_popen = lanes.subprocess.Popen
|
|
calls = []
|
|
|
|
def recording_popen(*args, **kwargs):
|
|
calls.append(kwargs)
|
|
return original_popen(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(lanes.subprocess, "Popen", recording_popen)
|
|
result = lanes.stream_process(
|
|
[sys.executable, "-c", "print('done')"],
|
|
provider="codex",
|
|
cwd=tmp_path,
|
|
env=dict(os.environ),
|
|
log_path=tmp_path / "worker.log",
|
|
state={},
|
|
state_file=tmp_path / "state.json",
|
|
heartbeat=lambda _note: True,
|
|
max_runtime=60,
|
|
)
|
|
|
|
assert result.returncode == 0
|
|
assert calls[0]["start_new_session"] is True
|
|
|
|
|
|
def test_worker_process_group_is_killed_after_leader_exits(monkeypatch):
|
|
class FinishedProcess:
|
|
pid = 4321
|
|
|
|
@staticmethod
|
|
def poll():
|
|
return 0
|
|
|
|
signals = []
|
|
monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig)))
|
|
|
|
lanes._terminate_worker_process(FinishedProcess())
|
|
|
|
assert signals == [(4321, signal.SIGKILL)]
|
|
|
|
|
|
def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch):
|
|
class RunningProcess:
|
|
pid = 4321
|
|
running = True
|
|
|
|
@classmethod
|
|
def poll(cls):
|
|
return None if cls.running else 0
|
|
|
|
@classmethod
|
|
def wait(cls, timeout):
|
|
assert timeout == 10
|
|
cls.running = False
|
|
return 0
|
|
|
|
group_signals = []
|
|
process_signals = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_descendant_processes",
|
|
lambda _pid: {5000: (5000, 12345)},
|
|
)
|
|
monkeypatch.setattr(lanes, "_process_identity_matches", lambda _pid, _start: True)
|
|
monkeypatch.setattr(
|
|
lanes.os,
|
|
"killpg",
|
|
lambda process_group, sig: group_signals.append((process_group, sig)),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes.os,
|
|
"kill",
|
|
lambda pid, sig: process_signals.append((pid, sig)),
|
|
)
|
|
|
|
lanes._terminate_worker_process(RunningProcess())
|
|
|
|
assert set(group_signals) == {
|
|
(4321, signal.SIGTERM),
|
|
(5000, signal.SIGTERM),
|
|
(4321, signal.SIGKILL),
|
|
(5000, signal.SIGKILL),
|
|
}
|
|
assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)]
|
|
|
|
|
|
def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
|
|
task = SimpleNamespace(id="t_auto", assignee=None, status="ready")
|
|
assigned = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def assign_task(_conn, task_id, profile):
|
|
assigned.append((task_id, profile))
|
|
task.assignee = profile
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
recompute_ready=lambda _conn: None,
|
|
list_tasks=lambda _conn: [task],
|
|
assign_task=assign_task,
|
|
get_task=lambda _conn, _task_id: task,
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")]
|
|
assert assigned == [("t_auto", "cli-auto")]
|
|
|
|
|
|
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
|
|
class CorruptBoardError(Exception):
|
|
pass
|
|
|
|
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def connect(*, board):
|
|
if board == "cassandra":
|
|
raise CorruptBoardError("integrity_check failed")
|
|
return Connection()
|
|
|
|
fake_db = SimpleNamespace(
|
|
KanbanDbCorruptError=CorruptBoardError,
|
|
list_boards=lambda include_archived=False: [
|
|
{"slug": "cassandra"},
|
|
{"slug": "healthy"},
|
|
],
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=connect,
|
|
recompute_ready=lambda _conn: None,
|
|
list_tasks=lambda _conn: [task],
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
|
|
|
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
|
assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err
|
|
|
|
|
|
def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys):
|
|
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
|
|
|
class Connection:
|
|
def __init__(self, board):
|
|
self.board = board
|
|
|
|
def close(self):
|
|
return None
|
|
|
|
def recompute_ready(connection):
|
|
if connection.board == "cassandra":
|
|
raise lanes.sqlite3.OperationalError("disk I/O error")
|
|
|
|
fake_db = SimpleNamespace(
|
|
list_boards=lambda include_archived=False: [
|
|
{"slug": "cassandra"},
|
|
{"slug": "healthy"},
|
|
],
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(board),
|
|
recompute_ready=recompute_ready,
|
|
list_tasks=lambda _conn: [task],
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
|
|
|
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
|
error = capsys.readouterr().err
|
|
assert "temporarily skipping Kanban board 'cassandra'" in error
|
|
assert "storage OperationalError: disk I/O error" in error
|
|
|
|
|
|
def test_board_call_retries_storage_faults_on_fresh_connections():
|
|
connections = []
|
|
|
|
class Connection:
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
def connect(*, board):
|
|
assert board == "cassandra"
|
|
connection = Connection()
|
|
connections.append(connection)
|
|
return connection
|
|
|
|
attempts = []
|
|
|
|
def operation(_connection):
|
|
attempts.append(1)
|
|
if len(attempts) < 3:
|
|
raise lanes.sqlite3.OperationalError("disk I/O error")
|
|
return "healthy"
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=connect,
|
|
)
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
|
|
|
assert lanes._board_call(fake_db, "cassandra", operation) == "healthy"
|
|
assert len(connections) == 3
|
|
assert all(connection.closed for connection in connections)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("result", "expected_action"),
|
|
[
|
|
(lanes.ProcessResult(0, "plain text only", None, False), "block"),
|
|
(
|
|
lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q"],
|
|
"artifacts": ["reports/result.json"],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
"complete",
|
|
),
|
|
(
|
|
lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "The full test suite is still running.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q — in progress"],
|
|
"artifacts": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
"block",
|
|
),
|
|
],
|
|
)
|
|
def test_claim_requires_structured_evidence_and_surfaces_artifacts(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
result,
|
|
expected_action,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_worker",
|
|
current_run_id=4,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
calls = []
|
|
heartbeats = []
|
|
connections = []
|
|
artifact = tmp_path / "reports/result.json"
|
|
artifact.parent.mkdir()
|
|
artifact.write_text("{}\n", encoding="utf-8")
|
|
|
|
class Connection:
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
def connect(*, board):
|
|
assert board == "cassandra"
|
|
connection = Connection()
|
|
connections.append(connection)
|
|
return connection
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=connect,
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "bounded objective",
|
|
heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: (
|
|
heartbeats.append((note, expected_run_id)) or True
|
|
),
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
),
|
|
)
|
|
def run_provider(*args, **_kwargs):
|
|
assert connections[0].closed
|
|
before_heartbeat = len(connections)
|
|
assert args[6]("working") is True
|
|
assert len(connections) == before_heartbeat + 1
|
|
assert connections[-1].closed
|
|
return result
|
|
|
|
monkeypatch.setattr(lanes, "run_provider", run_provider)
|
|
|
|
lanes.execute_claim("cassandra", "t_worker")
|
|
|
|
assert calls[0][0] == expected_action
|
|
assert heartbeats == [("working", 4)]
|
|
if expected_action == "complete":
|
|
assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)]
|
|
assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"]
|
|
else:
|
|
assert calls[0][1]["kind"] == "capability"
|
|
assert all(connection.closed for connection in connections)
|
|
|
|
|
|
def test_goal_card_continues_after_local_judge_rejects_progress(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_goal",
|
|
current_run_id=12,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=300,
|
|
goal_mode=True,
|
|
goal_max_turns=3,
|
|
)
|
|
calls = []
|
|
comments = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
|
complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"state_path",
|
|
lambda _board, _task_id: tmp_path / "state.json",
|
|
)
|
|
claude_low = lanes.Route(
|
|
"claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
|
|
)
|
|
codex_low = lanes.Route(
|
|
"codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
|
|
)
|
|
codex_xhigh = lanes.Route(
|
|
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
|
|
)
|
|
route_calls = []
|
|
|
|
def select_route(_prompt, assignee, **kwargs):
|
|
route_calls.append((assignee, kwargs))
|
|
if assignee == "cli-codex-low":
|
|
return codex_low
|
|
if len(route_calls) == 1:
|
|
return claude_low
|
|
return codex_xhigh
|
|
|
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
|
reports = [
|
|
lanes.ProcessResult(1, "authentication expired", None, True),
|
|
lanes.ProcessResult(
|
|
0,
|
|
"first turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "Focused tests passed.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest focused: passed"],
|
|
"artifacts": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
lanes.ProcessResult(
|
|
0,
|
|
"second turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "Full tests passed; commit pushed and remote HEAD verified.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest full: passed"],
|
|
"artifacts": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
verdicts = iter(
|
|
[
|
|
(False, "commit, push, and remote verification are missing"),
|
|
(True, "all explicit acceptance criteria have evidence"),
|
|
]
|
|
)
|
|
judge_contexts = []
|
|
|
|
def judge_goal_completion(objective, *_args, **_kwargs):
|
|
judge_contexts.append(objective)
|
|
return next(verdicts)
|
|
|
|
monkeypatch.setattr(
|
|
lanes.cli_lane_goal,
|
|
"judge_goal_completion",
|
|
judge_goal_completion,
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_goal")
|
|
|
|
assert calls[0][0] == "complete"
|
|
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
|
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
|
|
assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
|
|
assert route_calls[2][1]["exclude_provider"] == "claude"
|
|
assert "prior rejected reports" in judge_contexts[1]
|
|
assert "commit, push, and remote verification are missing" in judge_contexts[1]
|
|
assert reports == []
|
|
|
|
|
|
def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
|
|
calls = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw(
|
|
ValueError("not a Git repository")
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json")
|
|
|
|
lanes.execute_claim("cassandra", "t_bad_worktree")
|
|
|
|
assert calls[0]["kind"] == "capability"
|
|
assert calls[0]["expected_run_id"] == 7
|
|
assert "not a Git repository" in calls[0]["reason"]
|
|
|
|
|
|
def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path):
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
inside = workspace / "report.json"
|
|
outside = tmp_path / "auth.json"
|
|
inside.write_text("{}\n", encoding="utf-8")
|
|
outside.write_text("secret\n", encoding="utf-8")
|
|
|
|
assert lanes.workspace_artifacts(
|
|
workspace,
|
|
["report.json", str(outside), "missing.json"],
|
|
) == [str(inside)]
|
|
|
|
|
|
def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(
|
|
id="t_resume",
|
|
current_run_id=9,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
state_file = tmp_path / "state.json"
|
|
state_file.write_text(
|
|
json.dumps({"current_route": {"provider": "claude"}}),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8")
|
|
prompts = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "resume objective",
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda *_args, **_kwargs: None,
|
|
block_task=lambda *_args, **_kwargs: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"git_handoff",
|
|
lambda _workspace, output: f"HANDOFF:{output}",
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda _route, prompt, *_args, **_kwargs: (
|
|
prompts.append(prompt)
|
|
or lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done",
|
|
"changed_files": [],
|
|
"tests_run": [],
|
|
"artifacts": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
)
|
|
),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_resume")
|
|
|
|
assert "HANDOFF:prior provider evidence" in prompts[0]
|
|
|
|
|
|
def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path):
|
|
route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ())
|
|
command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json")
|
|
assert "--dangerously-bypass-approvals-and-sandbox" in command
|
|
assert "--json" in command
|
|
assert "--output-schema" in command
|
|
assert 'model_reasoning_effort="xhigh"' in command
|
|
|
|
claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"}
|
|
claude = lanes._claude_command(
|
|
lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()),
|
|
"Review.",
|
|
claude_state,
|
|
False,
|
|
)
|
|
assert "--dangerously-skip-permissions" in claude
|
|
assert "--output-format" in claude and "stream-json" in claude
|
|
assert "--json-schema" in claude
|
|
assert "--disallowedTools" in claude
|
|
assert "Bash(kubectl apply *)" not in claude
|
|
assert "Bash(flux reconcile *)" not in claude
|
|
assert "max" not in claude
|
|
|
|
|
|
def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path):
|
|
prompt = lanes.build_prompt("Review the change.", tmp_path)
|
|
|
|
assert "put defects and risks in findings" in prompt
|
|
assert "blockers array must be empty whenever status is completed" in prompt
|
|
assert "findings" in lanes.RESULT_SCHEMA["properties"]
|
|
assert set(lanes.RESULT_SCHEMA["required"]) == set(
|
|
lanes.RESULT_SCHEMA["properties"]
|
|
)
|
|
assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"command",
|
|
[
|
|
"git push --force origin main",
|
|
"git reset --hard HEAD~1",
|
|
"git clean -fd",
|
|
],
|
|
)
|
|
def test_claude_pretool_hook_blocks_hard_denies(command: str):
|
|
assert policy.denial_reason(command)
|
|
|
|
|
|
def test_claude_pretool_hook_allows_normal_engineering():
|
|
assert policy.denial_reason("pytest -q testing/tests") is None
|
|
assert policy.denial_reason("git push origin feature/hermes") is None
|
|
assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None
|
|
assert policy.denial_reason("flux reconcile kustomization hermes") is None
|
|
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
|
|
|
|
|
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path):
|
|
state = tmp_path / ".claude.json"
|
|
settings = tmp_path / "settings.json"
|
|
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
|
settings.write_text(
|
|
json.dumps(
|
|
{
|
|
"theme": "dark",
|
|
"permissions": {
|
|
"deny": [
|
|
"Bash(kubectl apply *)",
|
|
"Bash(flux reconcile *)",
|
|
"Bash(vault kv *)",
|
|
"Bash(custom-owner-rule *)",
|
|
]
|
|
},
|
|
}
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
client_config.configure_claude_state(state)
|
|
client_config.configure_claude_settings(settings)
|
|
|
|
state_value = json.loads(state.read_text())
|
|
settings_value = json.loads(settings.read_text())
|
|
assert state_value["promptQueueUseCount"] == 4
|
|
assert state_value["bypassPermissionsModeAccepted"] is True
|
|
assert settings_value["theme"] == "dark"
|
|
assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"]
|
|
assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"]
|
|
assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"]
|
|
assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"]
|
|
assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"]
|
|
hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0]
|
|
assert "claude_command_policy.py" in hook["command"]
|
|
|
|
|
|
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
|
session = tmp_path / "home/.config/herdr/session.json"
|
|
session.parent.mkdir(parents=True)
|
|
session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8")
|
|
binary = tmp_path / "tools/bin/herdr"
|
|
binary.parent.mkdir(parents=True)
|
|
binary.write_text("legacy", encoding="utf-8")
|
|
(tmp_path / "home/.claude").mkdir()
|
|
(tmp_path / "home/.codex").mkdir()
|
|
|
|
archive = migration.archive_legacy_state(tmp_path)
|
|
|
|
value = json.loads(archive.read_text())
|
|
assert value["legacy_session"]["agents"][0]["session_id"] == "abc"
|
|
assert (tmp_path / "home/.claude").is_dir()
|
|
assert (tmp_path / "home/.codex").is_dir()
|
|
assert not binary.exists()
|
|
assert not (tmp_path / "home/.config/herdr").exists()
|
|
|
|
|
|
def test_agent_uses_one_native_kanban_control_plane():
|
|
configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
|
|
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
|
assert config["model"] == {
|
|
"provider": "atlas-switchyard",
|
|
"default": "atlas/auto/balanced",
|
|
"model": "atlas/auto/balanced",
|
|
}
|
|
assert config["kanban"]["dispatch_in_gateway"] is True
|
|
assert config["kanban"]["default_assignee"] == "cli-auto"
|
|
assert config["plugins"]["enabled"] == ["auto-router"]
|
|
|
|
deployment = _agent_deployment()
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
assert pod["enableServiceLinks"] is False
|
|
names = {item["name"] for item in pod["containers"]}
|
|
assert "cli-lane-runner" in names
|
|
assert "terminal" in names
|
|
assert not any("herdr" in name for name in names)
|
|
rendered = (HERMES / "agent-deployment.yaml").read_text()
|
|
assert "herdr server" not in rendered
|
|
assert "herdr-dispatch" not in rendered
|
|
|
|
|
|
def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
|
|
deployment = _agent_deployment()
|
|
containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
lane = containers["cli-lane-runner"]
|
|
environment = {item["name"]: item["value"] for item in lane["env"]}
|
|
|
|
assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
|
|
assert lane["resources"] == {
|
|
"requests": {"cpu": "100m", "memory": "256Mi"},
|
|
"limits": {"cpu": "2", "memory": "6Gi"},
|
|
}
|
|
|
|
|
|
def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
|
|
"""Placement correction: keep the agent off nodes that cannot hold it.
|
|
|
|
titan-04 is cordoned after repeated kernel undervoltage and kubelet
|
|
failure, and titan-19 was probe/Longhorn unstable under worker load, so
|
|
both must join the existing hard exclusions. That leaves titan-05 as the
|
|
healthy candidate, which is tight enough on requested CPU that the main
|
|
container has to give back 50m to schedule there.
|
|
"""
|
|
pod = _agent_deployment()["spec"]["template"]["spec"]
|
|
hostnames = next(
|
|
item
|
|
for item in pod["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"][0]["matchExpressions"]
|
|
if item["key"] == "kubernetes.io/hostname"
|
|
)
|
|
|
|
assert hostnames["operator"] == "NotIn"
|
|
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
|
|
|
hermes = next(
|
|
item for item in pod["containers"] if item["name"] == "hermes"
|
|
)
|
|
assert hermes["resources"]["requests"]["cpu"] == "300m"
|
|
|
|
|
|
def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
|
deployment = _agent_deployment()
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
containers = {item["name"]: item for item in pod["containers"]}
|
|
assert "webui" not in containers
|
|
|
|
assert "dashboard" not in containers
|
|
hermes = containers["hermes"]
|
|
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
|
assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600"
|
|
assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600"
|
|
assert hermes["command"] == ["/bin/sh", "-ec"]
|
|
startup = hermes["args"][0]
|
|
assert ". /opt/data/.env" in startup
|
|
assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup
|
|
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
|
assert hermes_env["HERMES_DASHBOARD"] == "1"
|
|
assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1"
|
|
assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119"
|
|
assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
|
assert hermes["securityContext"]["runAsUser"] == 0
|
|
assert hermes["securityContext"]["runAsGroup"] == 0
|
|
for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"):
|
|
probe = hermes[probe_name]
|
|
assert probe["exec"]["command"] == [
|
|
"curl",
|
|
"-fsS",
|
|
"http://127.0.0.1:9119/api/status",
|
|
]
|
|
|
|
terminal = containers["terminal"]
|
|
command = terminal["args"][0]
|
|
assert "--base-path /terminal" in command
|
|
assert "--check-origin" not in command
|
|
assert "/usr/bin/tmux new-session -A" in command
|
|
assert "--continue" in command
|
|
assert "--yolo" in command
|
|
terminal_env = {item["name"]: item["value"] for item in terminal["env"]}
|
|
assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
|
|
|
claude_broker = containers["claude-broker"]
|
|
claude_env = {item["name"]: item["value"] for item in claude_broker["env"]}
|
|
assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2"
|
|
assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
|
assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
|
|
|
args = containers["oauth2-proxy"]["args"]
|
|
terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/"
|
|
dashboard_upstream = "--upstream=http://127.0.0.1:9119/"
|
|
assert terminal_upstream in args
|
|
assert dashboard_upstream in args
|
|
assert args.index(terminal_upstream) < args.index(dashboard_upstream)
|
|
assert "--pass-host-header=false" in args
|
|
assert "--cookie-refresh=19m" in args
|
|
assert "--session-store-type=redis" in args
|
|
assert any(
|
|
arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
|
|
for arg in args
|
|
)
|
|
|
|
patch_init = next(
|
|
item for item in pod["initContainers"]
|
|
if item["name"] == "patch-tui-gateway"
|
|
)
|
|
assert patch_init["command"][-1] == "/patched/server.py"
|
|
for name in ("hermes", "terminal"):
|
|
mounts = containers[name]["volumeMounts"]
|
|
assert {
|
|
"name": "tui-gateway-patch",
|
|
"mountPath": "/opt/hermes/tui_gateway/server.py",
|
|
"subPath": "server.py",
|
|
} in mounts
|
|
|
|
ingress_documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text())
|
|
if item
|
|
]
|
|
middlewares = {
|
|
item["metadata"]["name"]: item
|
|
for item in ingress_documents
|
|
if item["kind"] == "Middleware"
|
|
}
|
|
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
|
"replacement"
|
|
].endswith("/terminal/")
|
|
assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
|
"customRequestHeaders"
|
|
]["Origin"] == "http://127.0.0.1:9119"
|
|
ingresses = {
|
|
item["metadata"]["name"]: item
|
|
for item in ingress_documents
|
|
if item["kind"] == "Ingress"
|
|
}
|
|
assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
|
] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
|
assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
|
] == "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
|
|
|
|
|
def test_broker_services_survive_sibling_container_readiness_loss():
|
|
services = _services()
|
|
|
|
for name in (
|
|
"hermes-image-broker",
|
|
"hermes-codex-broker",
|
|
"hermes-local-image",
|
|
"hermes-claude-broker",
|
|
):
|
|
assert services[name]["spec"]["publishNotReadyAddresses"] is True
|
|
|
|
|
|
def test_agent_dashboard_reconnects_all_transient_websockets():
|
|
dockerfile = (
|
|
HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent"
|
|
).read_text(encoding="utf-8")
|
|
assert "eventsRetryAttempt.current" in dockerfile
|
|
assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
|
|
assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
|
|
assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile
|
|
assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile
|
|
assert dockerfile.count("' await api.getSessions(1, 0,") == 2
|
|
assert ".then(() => gw.connect())" in dockerfile
|
|
assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile
|
|
assert "dashboard token rotated by a server restart" in dockerfile
|
|
|
|
|
|
def test_agent_refreshes_routes_after_restoring_cli_logins():
|
|
deployment = _agent_deployment()
|
|
init_containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
|
}
|
|
configure = init_containers["configure-agent-clients"]
|
|
command = configure["command"][-1]
|
|
assert "configure_agent_clients.py" in command
|
|
assert command.index("configure_agent_clients.py") < command.index(
|
|
"hermes_coordinator.py --once"
|
|
)
|
|
env = {item["name"]: item["value"] for item in configure["env"]}
|
|
assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
|
|
assert env["PYTHONPATH"] == "/opt/hermes"
|
|
for name in ("bootstrap-coordinator", "configure-agent-clients"):
|
|
route_env = {
|
|
item["name"]: item["value"]
|
|
for item in init_containers[name]["env"]
|
|
}
|
|
assert route_env["CODEX_HOME"] == "/runtime-access/codex"
|
|
assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
|
|
assert "/opt/data/tools/bin" in route_env["PATH"]
|
|
|
|
assert "patch-web-session-activity" in init_containers
|
|
web_patch = init_containers["patch-web-session-activity"]
|
|
assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"]
|
|
|
|
containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
steward_env = {
|
|
item["name"]: item["value"]
|
|
for item in containers["model-steward"]["env"]
|
|
}
|
|
assert steward_env["CODEX_HOME"] == "/runtime-access/codex"
|
|
assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
|
|
assert "/opt/data/tools/bin" in steward_env["PATH"]
|
|
hermes_mounts = {
|
|
(item["name"], item["mountPath"], item.get("subPath"))
|
|
for item in containers["hermes"]["volumeMounts"]
|
|
}
|
|
assert (
|
|
"web-server-patch",
|
|
"/opt/hermes/hermes_cli/web_server.py",
|
|
"web_server.py",
|
|
) in hermes_mounts
|
|
|
|
|
|
def test_flux_health_checks_follow_the_owner_oauth_sidecar():
|
|
flux = yaml.safe_load(FLUX_HERMES.read_text())
|
|
checks = {
|
|
(item["kind"], item["name"])
|
|
for item in flux["spec"]["healthChecks"]
|
|
}
|
|
assert ("Deployment", "hermes-agent") in checks
|
|
assert ("DaemonSet", "hermes-node-ssh-access") in checks
|
|
assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks
|
|
|
|
|
|
def test_agent_auth_is_bstein_group_and_email_bounded():
|
|
deployment = _agent_deployment()
|
|
oauth = next(
|
|
item for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
if item["name"] == "oauth2-proxy"
|
|
)
|
|
args = oauth["args"]
|
|
assert "--user-id-claim=sub" in args
|
|
assert "--oidc-groups-claim=groups" in args
|
|
assert "--allowed-group=/hermes-owner" in args
|
|
assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args
|
|
script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text()
|
|
assert 'group_name="hermes-owner"' in script
|
|
assert "username=bstein&exact=true" in script
|
|
assert '"full.path":"true"' in script
|
|
|
|
|
|
def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces():
|
|
documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
|
if item
|
|
]
|
|
isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation")
|
|
assert isolation["spec"]["ingress"] == [
|
|
{
|
|
"from": [
|
|
{
|
|
"namespaceSelector": {
|
|
"matchLabels": {
|
|
"kubernetes.io/metadata.name": "traefik"
|
|
}
|
|
},
|
|
"podSelector": {
|
|
"matchLabels": {"app.kubernetes.io/name": "traefik"}
|
|
},
|
|
}
|
|
],
|
|
"ports": [{"protocol": "TCP", "port": 4180}],
|
|
},
|
|
{
|
|
"from": [
|
|
{
|
|
"podSelector": {
|
|
"matchLabels": {"app": "hermes-chat-tenant"}
|
|
}
|
|
}
|
|
],
|
|
"ports": [
|
|
{"protocol": "TCP", "port": 9002},
|
|
{"protocol": "TCP", "port": 9003},
|
|
],
|
|
},
|
|
{
|
|
"from": [
|
|
{
|
|
"podSelector": {
|
|
"matchLabels": {"app": "hermes-switchyard"}
|
|
}
|
|
}
|
|
],
|
|
"ports": [
|
|
{"protocol": "TCP", "port": 9003},
|
|
{"protocol": "TCP", "port": 9006},
|
|
],
|
|
},
|
|
{
|
|
"from": [
|
|
{
|
|
"namespaceSelector": {
|
|
"matchLabels": {
|
|
"kubernetes.io/metadata.name": "monitoring"
|
|
}
|
|
},
|
|
"podSelector": {"matchLabels": {"app": "server"}},
|
|
}
|
|
],
|
|
"ports": [{"protocol": "TCP", "port": 9010}],
|
|
},
|
|
]
|
|
assert isolation["spec"]["egress"] == [{}]
|
|
|
|
|
|
def test_owner_agent_has_cluster_admin_kubernetes_context():
|
|
config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text())
|
|
assert config["current-context"] == "atlas-owner"
|
|
assert config["contexts"][0]["context"]["namespace"] == "default"
|
|
rbac_path = HERMES / "agent-rbac.yaml"
|
|
documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item]
|
|
binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding")
|
|
assert binding["roleRef"] == {
|
|
"apiGroup": "rbac.authorization.k8s.io",
|
|
"kind": "ClusterRole",
|
|
"name": "cluster-admin",
|
|
}
|
|
assert binding["subjects"] == [
|
|
{"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
|
|
]
|
|
|
|
|
|
def test_owner_agent_has_pinned_dedicated_node_ssh_access():
|
|
deployment = _agent_deployment()
|
|
annotations = deployment["spec"]["template"]["metadata"]["annotations"]
|
|
assert annotations[
|
|
"vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"
|
|
] == "kv/data/atlas/hermes/developer-ssh"
|
|
assert annotations[
|
|
"vault.hashicorp.com/agent-inject-secret-node-ssh-config"
|
|
] == "kv/data/atlas/hermes/developer-ssh"
|
|
assert annotations[
|
|
"vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"
|
|
] == "kv/data/atlas/hermes/developer-ssh"
|
|
|
|
init = next(
|
|
item
|
|
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
|
if item["name"] == "init-config"
|
|
)
|
|
command = init["command"][2]
|
|
assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command
|
|
assert (
|
|
"ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts"
|
|
in command
|
|
)
|
|
assert "ln -s home/.ssh /opt/data/.ssh" in command
|
|
assert "chmod 0700 /opt/data/home/.ssh" in command
|
|
assert (
|
|
"ln -s /runtime-access/node-ssh-private-key "
|
|
"/opt/data/home/.ssh/id_ed25519_atlas_nodes"
|
|
) in command
|
|
|
|
config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"]
|
|
assert "ssh_config" not in config
|
|
assert "ssh_known_hosts" not in config
|
|
assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text()
|
|
|
|
resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[
|
|
"resources"
|
|
]
|
|
assert "node-ssh-access.yaml" in resources
|
|
access = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text())
|
|
if item
|
|
]
|
|
service_account = next(item for item in access if item["kind"] == "ServiceAccount")
|
|
assert service_account["metadata"]["name"] == "hermes-node-ssh-access"
|
|
provider = next(item for item in access if item["kind"] == "SecretProviderClass")
|
|
assert provider["metadata"]["name"] == "hermes-node-ssh-access"
|
|
assert provider["spec"]["provider"] == "vault"
|
|
parameters = provider["spec"]["parameters"]
|
|
assert parameters["roleName"] == "hermes-node-ssh"
|
|
assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"]
|
|
assert 'secretKey: "public_key"' in parameters["objects"]
|
|
daemonset = next(item for item in access if item["kind"] == "DaemonSet")
|
|
pod = daemonset["spec"]["template"]["spec"]
|
|
assert pod["serviceAccountName"] == "hermes-node-ssh-access"
|
|
assert pod["automountServiceAccountToken"] is True
|
|
host_home = next(item for item in pod["volumes"] if item["name"] == "host-home")
|
|
assert host_home["hostPath"] == {"path": "/home", "type": "Directory"}
|
|
vault_secrets = next(
|
|
item for item in pod["volumes"] if item["name"] == "vault-secrets"
|
|
)
|
|
assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io"
|
|
assert vault_secrets["csi"]["volumeAttributes"] == {
|
|
"secretProviderClass": "hermes-node-ssh-access"
|
|
}
|
|
reconciler = pod["containers"][0]["args"][0]
|
|
assert "cat /vault/secrets/node-ssh-public-key" in reconciler
|
|
assert "grep -qxF" in reconciler
|
|
assert "for user in atlas oceanus" in reconciler
|
|
assert "/host-etc/passwd" in reconciler
|
|
assert "chown \"${uid}:${gid}\"" in reconciler
|
|
host_passwd = next(
|
|
item for item in pod["volumes"] if item["name"] == "host-passwd"
|
|
)
|
|
assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"}
|
|
|
|
|
|
def test_owner_agent_tracks_no_ssh_identity_or_host_key_material():
|
|
"""Vault references may be tracked; SSH identities and trust data may not."""
|
|
forbidden = (
|
|
"BEGIN OPENSSH PRIVATE KEY",
|
|
"ssh-ed25519 AAAA",
|
|
"ssh-rsa AAAA",
|
|
"IdentityFile ",
|
|
"UserKnownHostsFile ",
|
|
"StrictHostKeyChecking ",
|
|
"ssh_config:",
|
|
"ssh_known_hosts:",
|
|
)
|
|
text_suffixes = {
|
|
".conf",
|
|
".json",
|
|
".md",
|
|
".py",
|
|
".sh",
|
|
".toml",
|
|
".yaml",
|
|
".yml",
|
|
}
|
|
tracked = "\n".join(
|
|
path.read_text(encoding="utf-8")
|
|
for path in HERMES.rglob("*")
|
|
if path.is_file() and path.suffix in text_suffixes
|
|
)
|
|
for marker in forbidden:
|
|
assert marker not in tracked
|
|
|
|
|
|
def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog():
|
|
"""Routing must not inherit the owner agent's cluster-admin capability."""
|
|
service_accounts = [
|
|
item
|
|
for item in yaml.safe_load_all(
|
|
(HERMES / "vault-serviceaccount.yaml").read_text()
|
|
)
|
|
if item
|
|
]
|
|
assert any(
|
|
item["kind"] == "ServiceAccount"
|
|
and item["metadata"]["name"] == "hermes-switchyard"
|
|
for item in service_accounts
|
|
)
|
|
|
|
switchyard = yaml.safe_load(
|
|
(HERMES / "switchyard-deployment.yaml").read_text()
|
|
)
|
|
switchyard_pod = switchyard["spec"]["template"]["spec"]
|
|
assert switchyard_pod["serviceAccountName"] == "hermes-switchyard"
|
|
agent_pod = _agent_deployment()["spec"]["template"]["spec"]
|
|
for pod, container_name in (
|
|
(switchyard_pod, "worker-route-broker"),
|
|
(agent_pod, "claude-broker"),
|
|
):
|
|
container = next(item for item in pod["containers"] if item["name"] == container_name)
|
|
catalog = next(
|
|
item
|
|
for item in container["volumeMounts"]
|
|
if item["mountPath"] == "/routing-catalog"
|
|
)
|
|
assert catalog["readOnly"] is True
|
|
|
|
rbac = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text())
|
|
if item
|
|
]
|
|
binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding")
|
|
assert binding["subjects"] == [
|
|
{"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
|
|
]
|
|
|
|
|
|
def test_switchyard_active_state_uses_a_relocatable_rwx_claim():
|
|
"""A stale node attachment must not strand the routing authority."""
|
|
claims = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text())
|
|
if item
|
|
]
|
|
active_claim = next(
|
|
item
|
|
for item in claims
|
|
if item["metadata"]["name"] == "hermes-switchyard-state-rwx"
|
|
)
|
|
assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"]
|
|
|
|
deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text())
|
|
strategy = deployment["spec"]["strategy"]
|
|
assert strategy == {
|
|
"type": "RollingUpdate",
|
|
"rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0},
|
|
}
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
state = next(item for item in pod["volumes"] if item["name"] == "state")
|
|
assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][
|
|
"name"
|
|
]
|
|
|
|
|
|
def test_worker_route_broker_accepts_pod_network_health_checks():
|
|
"""Kubelet probes the pod IP, so the broker cannot bind to loopback only."""
|
|
script = (SCRIPTS / "worker_route_broker.py").read_text()
|
|
assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script
|
|
|
|
|
|
def test_switchyard_network_boundary_allows_vault_bootstrap():
|
|
"""The pre-populate init container must reach Vault before routing starts."""
|
|
documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
|
if item
|
|
]
|
|
isolation = next(
|
|
item
|
|
for item in documents
|
|
if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
|
|
)
|
|
assert any(
|
|
rule.get("to")
|
|
== [
|
|
{
|
|
"namespaceSelector": {
|
|
"matchLabels": {"kubernetes.io/metadata.name": "vault"}
|
|
},
|
|
"podSelector": {"matchLabels": {"app": "vault"}},
|
|
}
|
|
]
|
|
and rule.get("ports") == [{"protocol": "TCP", "port": 8200}]
|
|
for rule in isolation["spec"]["egress"]
|
|
)
|
|
|
|
|
|
def test_switchyard_network_boundary_allows_metrics_scraping():
|
|
"""VictoriaMetrics may scrape Switchyard without widening its API boundary."""
|
|
documents = [
|
|
item
|
|
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
|
if item
|
|
]
|
|
isolation = next(
|
|
item
|
|
for item in documents
|
|
if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
|
|
)
|
|
assert any(
|
|
rule.get("from")
|
|
== [
|
|
{
|
|
"namespaceSelector": {
|
|
"matchLabels": {
|
|
"kubernetes.io/metadata.name": "monitoring"
|
|
}
|
|
},
|
|
"podSelector": {"matchLabels": {"app": "server"}},
|
|
}
|
|
]
|
|
and rule.get("ports") == [{"protocol": "TCP", "port": 9005}]
|
|
for rule in isolation["spec"]["ingress"]
|
|
)
|
|
|
|
|
|
def test_owner_agent_installs_the_pinned_operator_toolchain():
|
|
script = (SCRIPTS / "install_agent_tools.sh").read_text()
|
|
for value in [
|
|
"flux",
|
|
"helm",
|
|
"kustomize",
|
|
"jq",
|
|
"yq",
|
|
"gh",
|
|
"vault",
|
|
"sops",
|
|
"age",
|
|
"age-keygen",
|
|
"k9s",
|
|
"terraform",
|
|
"go",
|
|
"gofmt",
|
|
]:
|
|
assert value in script
|
|
assert "go1.26.5.linux-arm64.tar.gz" in script
|
|
assert (
|
|
"fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49"
|
|
in script
|
|
)
|
|
assert script.count("sha256sum -c -") == 1
|
|
|
|
deployment = _agent_deployment()
|
|
installer = next(
|
|
item
|
|
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
|
if item["name"] == "install-agent-tools"
|
|
)
|
|
assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2]
|
|
assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"])
|
|
init_config = next(
|
|
item
|
|
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
|
if item["name"] == "init-config"
|
|
)
|
|
init_command = init_config["command"][2]
|
|
assert "# Hermes managed operator PATH." in init_command
|
|
assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command
|
|
assert 'chmod 0644 "${profile_file}"' in init_command
|
|
|
|
|
|
def test_owner_agent_uses_only_the_canonical_hostname():
|
|
paths = [
|
|
HERMES / "agent-configmap.yaml",
|
|
HERMES / "agent-deployment.yaml",
|
|
HERMES / "agent-ingress.yaml",
|
|
Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py",
|
|
]
|
|
for path in paths:
|
|
content = path.read_text()
|
|
assert "agent.bstein.dev" not in content
|
|
assert "agent.hermes.bstein.dev" in content
|
|
|
|
|
|
def test_agent_reconnect_retains_complete_history_and_long_tool_budget():
|
|
configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
|
|
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
|
display = config["display"]
|
|
assert display["resume_exchanges"] >= 10000
|
|
assert display["resume_max_user_chars"] >= 10000000
|
|
assert display["resume_max_assistant_chars"] >= 10000000
|
|
assert config["agent"]["max_turns"] == 180
|
|
assert config["delegation"]["max_iterations"] == 120
|
|
|
|
|
|
def test_auth_patch_honors_explicit_shared_store(tmp_path: Path):
|
|
source = tmp_path / "auth.py"
|
|
destination = tmp_path / "patched/auth.py"
|
|
source.write_text(
|
|
'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n',
|
|
encoding="utf-8",
|
|
)
|
|
auth_patch.patch(source, destination)
|
|
content = destination.read_text()
|
|
assert 'os.environ.get("HERMES_AUTH_FILE"' in content
|
|
|
|
|
|
def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path):
|
|
source = tmp_path / "auth.py"
|
|
source.write_text("def changed():\n pass\n", encoding="utf-8")
|
|
with pytest.raises(RuntimeError, match="context changed"):
|
|
auth_patch.patch(source, tmp_path / "patched.py")
|
|
|
|
|
|
def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path):
|
|
provider = tmp_path / "runtime_provider.py"
|
|
provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8")
|
|
provider_out = tmp_path / "patched/runtime_provider.py"
|
|
codex_runtime_patch.patch_provider(provider, provider_out)
|
|
assert '"api_mode": "codex_app_server"' in provider_out.read_text()
|
|
|
|
session = tmp_path / "codex_app_server_session.py"
|
|
session.write_text(
|
|
codex_runtime_patch.SESSION_SIGNATURE_BEFORE
|
|
+ codex_runtime_patch.SESSION_REQUEST_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
session_out = tmp_path / "patched/codex_app_server_session.py"
|
|
codex_runtime_patch.patch_session(session, session_out)
|
|
session_content = session_out.read_text()
|
|
assert 'turn_params["model"] = model' in session_content
|
|
assert 'turn_params["effort"] = effort' in session_content
|
|
assert '"approvalPolicy": "never"' in session_content
|
|
assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content
|
|
|
|
turn = tmp_path / "codex_runtime.py"
|
|
turn.write_text(
|
|
codex_runtime_patch.FALLBACK_CONTEXT_BEFORE
|
|
+ codex_runtime_patch.TURN_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
turn_out = tmp_path / "patched/codex_runtime.py"
|
|
codex_runtime_patch.patch_turn(turn, turn_out)
|
|
turn_content = turn_out.read_text()
|
|
assert "model=str(getattr(agent" in turn_content
|
|
assert "build_cross_provider_codex_prompt" in turn_content
|
|
|
|
fallback = tmp_path / "chat_completion_helpers.py"
|
|
fallback.write_text(
|
|
codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
fallback_out = tmp_path / "patched/chat_completion_helpers.py"
|
|
codex_runtime_patch.patch_fallback(fallback, fallback_out)
|
|
fallback_content = fallback_out.read_text()
|
|
assert 'agent.api_mode = "codex_app_server"' in fallback_content
|
|
assert "agent._codex_cross_provider_fallback = True" in fallback_content
|
|
|
|
loop = tmp_path / "conversation_loop.py"
|
|
loop.write_text(
|
|
codex_runtime_patch.FALLBACK_DISPATCH_BEFORE
|
|
+ codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE
|
|
+ codex_runtime_patch.STREAM_RECOVERY_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
loop_out = tmp_path / "patched/conversation_loop.py"
|
|
codex_runtime_patch.patch_loop(loop, loop_out)
|
|
loop_content = loop_out.read_text()
|
|
assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2
|
|
assert "build_cross_provider_codex_prompt" in loop_content
|
|
retry_dispatch = loop_content.index(
|
|
"Fallback activation happens inside this retry loop"
|
|
)
|
|
api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch)
|
|
assert api_kwargs == -1 or retry_dispatch < api_kwargs
|
|
assert "Provider stream ended before a complete response" in loop_content
|
|
assert "_is_transport_stub" in loop_content
|
|
assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content
|
|
|
|
auxiliary = tmp_path / "auxiliary_client.py"
|
|
auxiliary.write_text(
|
|
codex_runtime_patch.AUXILIARY_TOKEN_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
auxiliary_out = tmp_path / "patched/auxiliary_client.py"
|
|
codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out)
|
|
auxiliary_content = auxiliary_out.read_text()
|
|
assert 'os.environ.get("CODEX_HOME"' in auxiliary_content
|
|
assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content
|
|
assert "never creates a metered API-key lane" in auxiliary_content
|
|
|
|
|
|
def test_agent_mounts_codex_auxiliary_runtime_patch():
|
|
deployment = _agent_deployment()
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
patch_init = next(
|
|
item for item in pod["initContainers"]
|
|
if item["name"] == "patch-codex-runtime"
|
|
)
|
|
assert patch_init["command"][-2:] == [
|
|
"/opt/hermes/agent/auxiliary_client.py",
|
|
"/patched/auxiliary_client.py",
|
|
]
|
|
expected_mount = {
|
|
"name": "codex-runtime-patch",
|
|
"mountPath": "/opt/hermes/agent/auxiliary_client.py",
|
|
"subPath": "auxiliary_client.py",
|
|
}
|
|
containers = {item["name"]: item for item in pod["containers"]}
|
|
for name in ("hermes", "terminal"):
|
|
assert expected_mount in containers[name]["volumeMounts"]
|
|
|
|
|
|
def test_codex_auxiliary_patch_reads_cli_token_without_copying_it(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
codex_home = tmp_path / ".codex"
|
|
codex_home.mkdir()
|
|
(codex_home / "auth.json").write_text(
|
|
json.dumps({"tokens": {"access_token": "cli-access-token"}}),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("CODEX_HOME", str(codex_home))
|
|
|
|
source = tmp_path / "auxiliary_client.py"
|
|
source.write_text(
|
|
"import json, logging, os, time\n"
|
|
"from pathlib import Path\n"
|
|
"logger = logging.getLogger(__name__)\n"
|
|
"def read_token():\n"
|
|
" try:\n"
|
|
" raise RuntimeError('Hermes provider store intentionally empty')\n"
|
|
+ codex_runtime_patch.AUXILIARY_TOKEN_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
destination = tmp_path / "patched/auxiliary_client.py"
|
|
codex_runtime_patch.patch_auxiliary(source, destination)
|
|
namespace: dict = {}
|
|
exec(compile(destination.read_text(), str(destination), "exec"), namespace)
|
|
|
|
assert namespace["read_token"]() == "cli-access-token"
|
|
|
|
|
|
def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path):
|
|
source = tmp_path / "runtime_provider.py"
|
|
source.write_text("def changed():\n pass\n", encoding="utf-8")
|
|
with pytest.raises(RuntimeError, match="context changed"):
|
|
codex_runtime_patch.patch_provider(source, tmp_path / "patched.py")
|
|
|
|
|
|
def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path):
|
|
config = tmp_path / "config.yaml"
|
|
config.write_text("model: {}\n", encoding="utf-8")
|
|
codex_home = tmp_path / ".codex"
|
|
calls = []
|
|
|
|
class Report:
|
|
errors = []
|
|
|
|
@staticmethod
|
|
def summary():
|
|
return "configured"
|
|
|
|
def migrate(value, **kwargs):
|
|
calls.append((value, kwargs))
|
|
return Report()
|
|
|
|
client_config.configure_codex_runtime(config, migrate, codex_home)
|
|
|
|
assert calls[0][1]["default_permission_profile"] is None
|
|
assert calls[0][1]["codex_home"] == codex_home
|
|
content = (codex_home / "config.toml").read_text(encoding="utf-8")
|
|
assert 'approval_policy = "never"' in content
|
|
assert 'sandbox_mode = "danger-full-access"' in content
|
|
assert "default_permissions" not in content
|
|
|
|
|
|
def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path):
|
|
config = tmp_path / "config.toml"
|
|
config.write_text(
|
|
'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
client_config.configure_codex_owner_permissions(config)
|
|
client_config.configure_codex_owner_permissions(config)
|
|
|
|
content = config.read_text(encoding="utf-8")
|
|
assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1
|
|
assert content.count('approval_policy = "never"') == 1
|
|
assert content.count('sandbox_mode = "danger-full-access"') == 1
|
|
assert "default_permissions" not in content
|
|
assert "[features]\nhooks = true" in content
|
|
|
|
|
|
def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path):
|
|
source = tmp_path / "server.py"
|
|
destination = tmp_path / "patched/server.py"
|
|
source.write_text(
|
|
"import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
tui_gateway_patch.patch(source, destination)
|
|
|
|
content = destination.read_text(encoding="utf-8")
|
|
assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content
|
|
assert 'configured = 180.0' in content
|
|
assert "return max(30.0, min(configured, 900.0))" in content
|
|
assert "timeout: float | None = None" in content
|
|
assert "ready.wait(timeout=wait_timeout)" in content
|
|
assert "def unchanged():" in content
|
|
|
|
|
|
def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path):
|
|
source = tmp_path / "server.py"
|
|
source.write_text("def changed():\n pass\n", encoding="utf-8")
|
|
with pytest.raises(RuntimeError, match="context changed"):
|
|
tui_gateway_patch.patch(source, tmp_path / "patched.py")
|
|
|
|
|
|
def test_ttyd_clipboard_and_reconnect_patch_remain_enabled():
|
|
source = '<html><body><script>document.execCommand("copy")</script></body></html>'
|
|
content = ttyd_patch.patch_html(source)
|
|
assert 'id="atlas-ttyd-clipboard"' in content
|
|
assert "navigator.clipboard.writeText(text)" in content
|
|
assert "class AtlasRecoveringWebSocket" in content
|
|
assert "window.location.reload()" in content
|
|
assert "event.stopImmediatePropagation()" in content
|
|
|
|
|
|
def test_ttyd_patch_fails_closed_on_upstream_drift():
|
|
with pytest.raises(RuntimeError, match="context changed"):
|
|
ttyd_patch.patch_html("<html><body>changed</body></html>")
|