466 lines
18 KiB
Python
466 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import cli_lane_runner
|
|
from execution_pool_protocol import (
|
|
ProtocolError,
|
|
atomic_json,
|
|
canonical_json,
|
|
)
|
|
|
|
|
|
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
|
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
|
|
CLIENT = os.environ.get(
|
|
"HERMES_EXECUTION_CLIENT_URL",
|
|
f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009",
|
|
).rstrip("/")
|
|
NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
|
|
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
|
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
|
|
DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip()
|
|
|
|
|
|
def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
|
|
request = urllib.request.Request(
|
|
url, data=canonical_json(value), method="POST",
|
|
headers={"Content-Type": "application/json", "Cache-Control": "no-store"},
|
|
)
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read(64 * 1024 + 1)
|
|
if len(body) > 64 * 1024:
|
|
raise ProtocolError("server response exceeds the wire limit")
|
|
document = json.loads(body)
|
|
if not isinstance(document, dict):
|
|
raise ProtocolError("server response is not an object")
|
|
return document
|
|
|
|
|
|
def _client(operation: str, **values: Any) -> dict[str, Any]:
|
|
response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values})
|
|
if response.get("error"):
|
|
raise ProtocolError(str(response["error"]))
|
|
return response
|
|
|
|
|
|
def _poll() -> dict[str, Any] | None:
|
|
assignment = _client("poll").get("assignment")
|
|
if assignment is None:
|
|
return None
|
|
if (
|
|
not isinstance(assignment, dict)
|
|
or int(assignment["worker_ordinal"]) != ORDINAL
|
|
or int(assignment.get("protocol_version", 0)) != 2
|
|
):
|
|
raise ProtocolError("local boundary returned a foreign assignment")
|
|
return assignment
|
|
|
|
|
|
def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
|
|
return {name: assignment[name] for name in (
|
|
"board", "task_id", "run_id", "worker_ordinal", "attempt"
|
|
)}
|
|
|
|
|
|
def _state_path(assignment: dict[str, Any]) -> Path:
|
|
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
|
if any(not RUN_PART.fullmatch(part) for part in parts):
|
|
raise ProtocolError("assignment state path is invalid")
|
|
configured_root = ROOT / "session-state"
|
|
if configured_root.is_symlink():
|
|
raise ProtocolError("assignment state root must not be a symlink")
|
|
root = configured_root.resolve()
|
|
path = root.joinpath(*parts).with_suffix(".json")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent):
|
|
raise ProtocolError("assignment state path contains a symlink")
|
|
path.resolve(strict=False).relative_to(root)
|
|
return path
|
|
|
|
|
|
def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
|
|
"""Attach provider session directories to this exact durable task/run."""
|
|
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
|
if any(not RUN_PART.fullmatch(part) for part in parts):
|
|
raise ProtocolError("provider session binding is invalid")
|
|
configured_root = ROOT / "provider-state"
|
|
if configured_root.is_symlink() or not configured_root.is_dir():
|
|
raise ProtocolError("durable provider state root is unavailable")
|
|
provider_root = configured_root.resolve()
|
|
run_root = provider_root
|
|
for part in parts:
|
|
run_root /= part
|
|
if run_root.is_symlink():
|
|
raise ProtocolError("provider session path contains a symlink")
|
|
run_root.mkdir(mode=0o700, exist_ok=True)
|
|
run_root.resolve().relative_to(provider_root)
|
|
home = run_root / "home"
|
|
if home.is_symlink():
|
|
raise ProtocolError("provider HOME must not be a durable symlink")
|
|
home.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
runtime_home = cli_lane_runner.DATA_ROOT / "home"
|
|
if runtime_home.is_symlink():
|
|
runtime_home.unlink()
|
|
elif runtime_home.exists():
|
|
raise ProtocolError("provider HOME is not a task-bound symlink")
|
|
runtime_home.symlink_to(home)
|
|
(home / ".claude").mkdir(mode=0o700, exist_ok=True)
|
|
settings = home / ".claude/settings.json"
|
|
if not settings.exists():
|
|
atomic_json(settings, {}, 0o600)
|
|
bindings = {
|
|
Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "sessions":
|
|
run_root / "codex/sessions",
|
|
Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "projects":
|
|
run_root / "claude/projects",
|
|
Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "session-env":
|
|
run_root / "claude/session-env",
|
|
Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "todos":
|
|
run_root / "claude/todos",
|
|
}
|
|
for runtime, durable in bindings.items():
|
|
current = run_root
|
|
for part in durable.relative_to(run_root).parts:
|
|
current /= part
|
|
if current.is_symlink():
|
|
raise ProtocolError("provider session path contains a symlink")
|
|
current.mkdir(mode=0o700, exist_ok=True)
|
|
current.resolve().relative_to(provider_root)
|
|
if runtime.is_symlink():
|
|
runtime.unlink()
|
|
elif runtime.exists():
|
|
raise ProtocolError(f"provider session path is not a symlink: {runtime.name}")
|
|
runtime.symlink_to(durable)
|
|
|
|
|
|
def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str:
|
|
return f"""You are a durable coding worker managed by Hermes Kanban.
|
|
|
|
Work only on this objective and its acceptance criteria:
|
|
|
|
{context}
|
|
|
|
Workspace: {workspace}
|
|
Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL}
|
|
|
|
Operate autonomously only inside this private assigned checkout. Inspect before editing,
|
|
preserve unrelated and untracked files, and fail closed rather than overwrite state you do
|
|
not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to
|
|
read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots, or bypass
|
|
the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch;
|
|
the worker boundary handles the bounded push and draft pull request after validation.
|
|
Switchyard owns provider/model/effort selection and cross-provider fallback.
|
|
|
|
Return a final JSON object matching the supplied schema. Use status=incomplete when work,
|
|
tests, commits, or verification remain. Use blocked only for a concrete task obstacle.
|
|
Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers.
|
|
"""
|
|
|
|
|
|
def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
|
|
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
|
|
try:
|
|
descriptor = os.open(log_path, flags)
|
|
except FileNotFoundError:
|
|
return "", offset
|
|
try:
|
|
info = os.fstat(descriptor)
|
|
if not stat.S_ISREG(info.st_mode):
|
|
raise ProtocolError("worker log must be a regular file")
|
|
if info.st_size < offset:
|
|
offset = 0
|
|
os.lseek(descriptor, offset, os.SEEK_SET)
|
|
value = os.read(descriptor, 12 * 1024)
|
|
next_offset = os.lseek(descriptor, 0, os.SEEK_CUR)
|
|
finally:
|
|
os.close(descriptor)
|
|
return value.decode("utf-8", "replace"), next_offset
|
|
|
|
|
|
def _git(workspace: Path, *arguments: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL,
|
|
text=True, capture_output=True, timeout=30, check=False,
|
|
env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "/bin/false"},
|
|
)
|
|
if completed.returncode:
|
|
raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:])
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
|
|
"""Keep terminal evidence useful while fitting the authenticated wire cap."""
|
|
bounded: dict[str, Any] = {}
|
|
for name in ("status", "summary"):
|
|
bounded[name] = str(value.get(name) or "")[:8000]
|
|
for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
|
|
items = value.get(name)
|
|
bounded[name] = [str(item)[:2000] for item in items[:100]] if isinstance(items, list) else []
|
|
while len(canonical_json(bounded)) > 32 * 1024:
|
|
longest = max(
|
|
(name for name in bounded if isinstance(bounded[name], list) and bounded[name]),
|
|
key=lambda name: len(canonical_json(bounded[name])),
|
|
default="",
|
|
)
|
|
if longest:
|
|
bounded[longest].pop()
|
|
else:
|
|
bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2]
|
|
return bounded
|
|
|
|
|
|
def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]:
|
|
fresh = _poll()
|
|
if fresh is None or _binding(fresh) != binding:
|
|
raise ProtocolError("assignment changed before SCM submission")
|
|
return fresh
|
|
|
|
|
|
def garbage_collect(now: float | None = None) -> int:
|
|
"""Remove only clean, terminal, assignment-derived workspaces after retention."""
|
|
current = time.time() if now is None else now
|
|
state_root = (ROOT / "session-state").resolve()
|
|
run_root = (ROOT / "runs").resolve()
|
|
removed = 0
|
|
for state_file in state_root.glob("*/*/*.json") if state_root.is_dir() else ():
|
|
state = cli_lane_runner.load_json(state_file)
|
|
terminal_at = float(state.get("terminal_at") or 0)
|
|
if terminal_at <= 0 or current - terminal_at < max(3600, RETENTION_SECONDS):
|
|
continue
|
|
workspace = Path(str(state.get("workspace") or ""))
|
|
try:
|
|
workspace.resolve(strict=True).relative_to(run_root)
|
|
except (OSError, RuntimeError, ValueError):
|
|
continue
|
|
if workspace.is_symlink() or _git(
|
|
workspace, "status", "--porcelain=v1", "--untracked-files=all"
|
|
):
|
|
continue
|
|
shutil.rmtree(workspace)
|
|
state_file.unlink()
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
def execute(assignment: dict[str, Any]) -> None:
|
|
binding = _binding(assignment)
|
|
payload = assignment["payload"]
|
|
if not isinstance(payload, dict):
|
|
raise ProtocolError("assignment payload is invalid")
|
|
_bind_provider_sessions(assignment)
|
|
workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
|
|
workspace.relative_to((ROOT / "runs").resolve())
|
|
state_file = _state_path(assignment)
|
|
state = cli_lane_runner.load_json(state_file)
|
|
state.update({**binding, "node": NODE, "workspace": str(workspace)})
|
|
state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or ""))
|
|
atomic_json(state_file, state)
|
|
log_path = state_file.with_suffix(".log")
|
|
offset = 0
|
|
latest_route: dict[str, Any] = {}
|
|
|
|
def heartbeat(note: str) -> bool:
|
|
nonlocal offset
|
|
try:
|
|
activity, offset = _read_activity(log_path, offset)
|
|
response = _client(
|
|
"heartbeat", binding=binding,
|
|
payload={
|
|
"note": f"worker={ORDINAL} node={NODE} {note}"[:1000],
|
|
"activity": activity, "route": latest_route,
|
|
},
|
|
)
|
|
return bool(response.get("ack", {}).get("accepted"))
|
|
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
|
return False
|
|
|
|
context = str(payload.get("context") or "")
|
|
assignee = str(payload.get("assignee") or "cli-auto")
|
|
excluded = None
|
|
if assignee == "cli-auto":
|
|
excluded = DISABLED_PROVIDER or cli_lane_runner.fresh_unavailable_provider()
|
|
route = cli_lane_runner.select_route(
|
|
context,
|
|
assignee,
|
|
exclude_provider=excluded,
|
|
)
|
|
latest_route = asdict(route)
|
|
if not heartbeat(
|
|
f"route={route.provider}/{route.model}/{route.effort}; assignment accepted"
|
|
):
|
|
raise ProtocolError("assignment lease was lost before provider startup")
|
|
deadline = min(
|
|
int(payload.get("deadline_unix") or int(time.time()) + 60),
|
|
int(time.time()) + int(payload.get("max_runtime_seconds") or 60),
|
|
)
|
|
result = cli_lane_runner.run_provider(
|
|
route, _prompt(context, workspace, binding), workspace, state, state_file,
|
|
log_path, heartbeat, max(1, deadline - int(time.time())),
|
|
)
|
|
if result.capacity_failure and deadline - time.time() > 30:
|
|
alternate = "claude" if route.provider == "codex" else "codex"
|
|
if alternate == DISABLED_PROVIDER:
|
|
alternate = ""
|
|
else:
|
|
alternate = ""
|
|
if alternate:
|
|
fallback = cli_lane_runner.select_route(
|
|
context + "\nThe first provider failed from capacity or authentication.",
|
|
f"cli-{alternate}-{route.effort}",
|
|
)
|
|
latest_route = asdict(fallback)
|
|
heartbeat(f"fallback={route.provider}->{fallback.provider}")
|
|
result = cli_lane_runner.run_provider(
|
|
fallback,
|
|
_prompt(context, workspace, binding)
|
|
+ cli_lane_runner.git_handoff(workspace, result.output),
|
|
workspace, state, state_file, log_path, heartbeat,
|
|
max(1, deadline - int(time.time())),
|
|
)
|
|
route = fallback
|
|
structured = dict(result.structured or {})
|
|
for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
|
|
if not isinstance(structured.get(name), list):
|
|
structured[name] = []
|
|
if structured.get("status") == "completed" and result.returncode == 0:
|
|
status = _git(workspace, "status", "--porcelain=v1", "--untracked-files=all")
|
|
if status:
|
|
structured["status"] = "incomplete"
|
|
structured["blockers"].append(
|
|
"Worker left uncommitted or untracked files; SCM submission failed closed."
|
|
)
|
|
else:
|
|
baseline = str(state.get("baseline_sha") or "")
|
|
if not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
|
|
raise RuntimeError("durable SCM baseline is missing or invalid")
|
|
_refresh_assignment(binding)
|
|
structured = _bounded_result(structured)
|
|
activity, _ = _read_activity(log_path, offset)
|
|
result_payload = {
|
|
"structured": structured,
|
|
"returncode": result.returncode,
|
|
"capacity_failure": result.capacity_failure,
|
|
"node": NODE,
|
|
"route": asdict(route),
|
|
"provider_sessions": {
|
|
"codex_thread_id": state.get("codex_thread_id"),
|
|
"claude_session_id": state.get("claude_session_id"),
|
|
},
|
|
"final_activity": activity,
|
|
}
|
|
response = _client(
|
|
"finish",
|
|
binding=binding,
|
|
payload=result_payload,
|
|
title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240],
|
|
body=json.dumps(structured, indent=2, sort_keys=True)[:12000],
|
|
)
|
|
if not response.get("ack", {}).get("accepted"):
|
|
raise ProtocolError("coordinator did not accept the terminal result")
|
|
state["terminal_at"] = time.time()
|
|
atomic_json(state_file, state)
|
|
|
|
|
|
def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
|
|
"""Surface one worker exception as an exact-run transient result."""
|
|
binding = _binding(assignment)
|
|
detail = f"{type(error).__name__}: {error}"[:1500]
|
|
structured = {
|
|
"status": "blocked",
|
|
"summary": "Distributed worker failed before a safe handoff.",
|
|
"changed_files": [],
|
|
"tests_run": [],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [detail],
|
|
}
|
|
payload = {
|
|
"structured": structured,
|
|
"returncode": 1,
|
|
# This is an execution-infrastructure failure, not an objective capability
|
|
# verdict. The exact-run result releases the ordinal for another task.
|
|
"capacity_failure": True,
|
|
"node": NODE,
|
|
"route": {},
|
|
"provider_sessions": {},
|
|
"final_activity": detail,
|
|
}
|
|
try:
|
|
return bool(
|
|
_client("finish", binding=binding, payload=payload).get("ack", {}).get(
|
|
"accepted"
|
|
)
|
|
)
|
|
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
|
return False
|
|
|
|
|
|
def readiness() -> None:
|
|
if ORDINAL not in range(3):
|
|
raise ProtocolError("worker ordinal must be 0, 1, or 2")
|
|
for path in (ROOT, ROOT / "provider-state", cli_lane_runner.DATA_ROOT):
|
|
if not path.is_dir() or not os.access(path, os.W_OK):
|
|
raise ProtocolError(f"durable worker path is not writable: {path}")
|
|
credentials = (
|
|
Path(
|
|
os.environ.get(
|
|
"CLAUDE_CODE_OAUTH_TOKEN_FILE", "/claude-oauth-access/token"
|
|
)
|
|
),
|
|
)
|
|
for credential in credentials:
|
|
if not credential.is_file() or not os.access(credential, os.R_OK):
|
|
raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}")
|
|
cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
atomic_json(
|
|
cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
|
|
)
|
|
try:
|
|
_poll()
|
|
except urllib.error.HTTPError as error:
|
|
# A replacement pod can overlap the prior ordinal's coordinator lease.
|
|
# The main loop already defers this transient conflict until ownership
|
|
# transfers, so keep the container alive instead of CrashLooping.
|
|
if error.code != 409:
|
|
raise
|
|
|
|
|
|
def main() -> int:
|
|
readiness()
|
|
while True:
|
|
assignment = None
|
|
try:
|
|
garbage_collect()
|
|
assignment = _poll()
|
|
if assignment is None:
|
|
time.sleep(5)
|
|
continue
|
|
execute(assignment)
|
|
except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error:
|
|
surfaced = assignment is not None and report_exception(assignment, error)
|
|
print(
|
|
f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: "
|
|
f"{type(error).__name__}: {error}",
|
|
flush=True,
|
|
)
|
|
time.sleep(10)
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
|
|
raise SystemExit(main())
|