From 4669320adf5de66f9d0183fe7ef6fb8c69048334 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 8 Aug 2026 23:24:58 -0300 Subject: [PATCH] feat(hermes): strengthen isolated chat reasoning --- dockerfiles/Dockerfile.hermes-agent | 126 ++++++++++++++++ dockerfiles/Dockerfile.hermes-chat-sandbox | 18 +++ dockerfiles/hermes-chat-sandbox-server.py | 136 +++++++++++++++++ dockerfiles/hermes-public-extract/__init__.py | 8 + dockerfiles/hermes-public-extract/plugin.yaml | 7 + dockerfiles/hermes-public-extract/provider.py | 142 ++++++++++++++++++ dockerfiles/hermes-python-sandbox-tool.py | 75 +++++++++ services/hermes/chat-configmap.yaml | 35 ++++- services/hermes/chat-sandbox.yaml | 132 ++++++++++++++++ services/hermes/chat-statefulset.yaml | 12 +- services/hermes/kustomization.yaml | 1 + services/hermes/networkpolicy.yaml | 99 ++++++++++++ testing/tests/test_hermes_chat_quality.py | 100 ++++++++++++ 13 files changed, 879 insertions(+), 12 deletions(-) create mode 100644 dockerfiles/Dockerfile.hermes-chat-sandbox create mode 100644 dockerfiles/hermes-chat-sandbox-server.py create mode 100644 dockerfiles/hermes-public-extract/__init__.py create mode 100644 dockerfiles/hermes-public-extract/plugin.yaml create mode 100644 dockerfiles/hermes-public-extract/provider.py create mode 100644 dockerfiles/hermes-python-sandbox-tool.py create mode 100644 services/hermes/chat-sandbox.yaml create mode 100644 testing/tests/test_hermes_chat_quality.py diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 611bcbb54..251aa6146 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -4,6 +4,10 @@ FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510 USER root +# Keep a credential-free search provider available for private chat tenants. +# Paid/provider-backed search remains selectable through normal Hermes config. +RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4 + # Keep dashboard chat sockets tied to the intended React mount and conversation. # A resumed conversation needs a different PTY attachment key from a fresh chat; # reconnects to that same conversation must keep using the same key. @@ -111,6 +115,119 @@ for before, after, label in ( path.write_text(source) PY +# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs. +# Upstream currently applies only statically declared model_routes there, so +# the UI can display one model while the gateway silently runs another. Honor +# trusted first-party provider selections and cap all chat reasoning at xhigh. +RUN python - <<'PY' +from pathlib import Path + +path = Path("/opt/hermes/gateway/platforms/api_server.py") +source = path.read_text() + +route_before = ''' def _resolve_route(self, model_alias: Any) -> Optional[Dict[str, Any]]: + """Return the model_routes entry for *model_alias*, or None.""" + if not self._model_routes or not isinstance(model_alias, str): + return None + return self._model_routes.get(model_alias) + +''' +route_after = route_before + ''' def _resolve_request_route(self, body: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Resolve a static route or a trusted WebUI provider/model selection.""" + route = self._resolve_route(body.get("model")) + if route is not None: + return route + + provider = body.get("provider") + model = body.get("model") + allowed_providers = {"openai-codex", "anthropic"} + if provider not in allowed_providers or not isinstance(model, str): + return None + model = model.strip() + if not model or len(model) > 128 or any( + char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:/+-" + for char in model + ): + return None + return {"provider": provider, "model": model} + +''' + +signature_before = ''' gateway_session_key: Optional[str] = None, + route: Optional[Dict[str, Any]] = None, + ) -> Any: +''' +signature_after = ''' gateway_session_key: Optional[str] = None, + route: Optional[Dict[str, Any]] = None, + reasoning_effort: Any = None, + ) -> Any: +''' + +reasoning_before = ''' runtime_kwargs = _resolve_runtime_agent_kwargs() + reasoning_config = GatewayRunner._load_reasoning_config() + model = _resolve_gateway_model() +''' +reasoning_after = ''' runtime_kwargs = _resolve_runtime_agent_kwargs() + reasoning_config = GatewayRunner._load_reasoning_config() + from hermes_constants import parse_reasoning_effort + + requested_reasoning = parse_reasoning_effort(reasoning_effort) + if requested_reasoning is not None: + reasoning_config = requested_reasoning + if reasoning_config and reasoning_config.get("effort") == "max": + reasoning_config = {"enabled": True, "effort": "xhigh"} + model = _resolve_gateway_model() +''' + +runs_route_before = ''' # Per-client model routing for /v1/runs (see model_routes). + route = self._resolve_route(body.get("model")) +''' +runs_route_after = ''' # Honor both static routes and the WebUI's trusted provider/model pick. + route = self._resolve_request_route(body) +''' + +runs_agent_before = ''' gateway_session_key=gateway_session_key, + route=route, + ) +''' +runs_agent_after = ''' gateway_session_key=gateway_session_key, + route=route, + reasoning_effort=body.get("reasoning_effort"), + ) +''' + +for before, after, label, count in ( + (route_before, route_after, "request route resolver", 1), + (signature_before, signature_after, "agent reasoning argument", 1), + (reasoning_before, reasoning_after, "reasoning clamp", 1), + (runs_route_before, runs_route_after, "runs route", 1), +): + if source.count(before) != count: + raise SystemExit( + f"Hermes API {label} patch context changed: expected {count}, " + f"found {source.count(before)}" + ) + source = source.replace(before, after, count) + +# The same argument tail appears in other handlers. Restrict replacement to +# the /v1/runs section so non-WebUI API surfaces retain upstream behavior. +runs_start = source.index(" async def _handle_runs(") +runs_source = source[runs_start:] +if runs_source.count(runs_agent_before) != 1: + raise SystemExit( + "Hermes API /v1/runs agent-call patch context changed: expected 1, " + f"found {runs_source.count(runs_agent_before)}" + ) +runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1) +source = source[:runs_start] + runs_source +path.write_text(source) +PY + +COPY dockerfiles/hermes-python-sandbox-tool.py /opt/hermes/tools/python_sandbox_tool.py +COPY dockerfiles/hermes-public-extract/__init__.py /opt/hermes/plugins/web/public_extract/__init__.py +COPY dockerfiles/hermes-public-extract/plugin.yaml /opt/hermes/plugins/web/public_extract/plugin.yaml +COPY dockerfiles/hermes-public-extract/provider.py /opt/hermes/plugins/web/public_extract/provider.py + COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate RUN cd /opt/hermes/web \ @@ -119,4 +236,13 @@ RUN cd /opt/hermes/web \ && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \ && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \ /opt/hermes/hermes_cli/dashboard_auth/middleware.py \ + && grep -Fq '_resolve_request_route' \ + /opt/hermes/gateway/platforms/api_server.py \ + && grep -Fq 'reasoning_effort=body.get("reasoning_effort")' \ + /opt/hermes/gateway/platforms/api_server.py \ + && /opt/hermes/.venv/bin/python -m py_compile \ + /opt/hermes/gateway/platforms/api_server.py \ + /opt/hermes/tools/python_sandbox_tool.py \ + /opt/hermes/plugins/web/public_extract/provider.py \ + && /opt/hermes/.venv/bin/python -c 'import ddgs' \ && chmod 0755 /opt/hermes/bin/hermes-session-migrate diff --git a/dockerfiles/Dockerfile.hermes-chat-sandbox b/dockerfiles/Dockerfile.hermes-chat-sandbox new file mode 100644 index 000000000..64931874a --- /dev/null +++ b/dockerfiles/Dockerfile.hermes-chat-sandbox @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 +# dockerfiles/Dockerfile.hermes-chat-sandbox +FROM python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + HOME=/workspace + +RUN groupadd --gid 20000 sandbox \ + && useradd --uid 20000 --gid 20000 --home-dir /workspace --no-create-home sandbox + +COPY --chown=20000:20000 dockerfiles/hermes-chat-sandbox-server.py /opt/sandbox/server.py + +USER 20000:20000 +WORKDIR /workspace +EXPOSE 9080 + +ENTRYPOINT ["python", "-I", "/opt/sandbox/server.py"] diff --git a/dockerfiles/hermes-chat-sandbox-server.py b/dockerfiles/hermes-chat-sandbox-server.py new file mode 100644 index 000000000..771e98b40 --- /dev/null +++ b/dockerfiles/hermes-chat-sandbox-server.py @@ -0,0 +1,136 @@ +"""Small credential-free Python execution service for Hermes chat tenants.""" + +from __future__ import annotations + +import json +import os +import resource +import signal +import subprocess +import tempfile +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +HOST = "0.0.0.0" +PORT = 9080 +MAX_REQUEST_BYTES = 160 * 1024 +MAX_CODE_BYTES = 128 * 1024 +MAX_OUTPUT_BYTES = 100 * 1024 +WORKSPACE = Path("/workspace") + + +def _child_limits() -> None: + """Apply conservative CPU, memory, process, file, and descriptor limits.""" + os.setsid() + resource.setrlimit(resource.RLIMIT_CPU, (35, 35)) + resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024,) * 2) + resource.setrlimit(resource.RLIMIT_NPROC, (32, 32)) + resource.setrlimit(resource.RLIMIT_FSIZE, (32 * 1024 * 1024,) * 2) + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + + +def _execute(code: str) -> dict[str, object]: + """Run one isolated Python subprocess and return bounded output.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", prefix="hermes-", dir="/tmp", delete=False + ) as script: + script.write(code) + script_path = script.name + + env = { + "HOME": str(WORKSPACE), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/bin:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + } + try: + process = subprocess.Popen( + ["python", "-I", "-B", script_path], + cwd=WORKSPACE, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=_child_limits, + ) + try: + stdout, stderr = process.communicate(timeout=45) + timed_out = False + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + return { + "success": process.returncode == 0 and not timed_out, + "exit_code": process.returncode, + "timed_out": timed_out, + "stdout": stdout[:MAX_OUTPUT_BYTES].decode("utf-8", errors="replace"), + "stderr": stderr[:MAX_OUTPUT_BYTES].decode("utf-8", errors="replace"), + "output_truncated": ( + len(stdout) > MAX_OUTPUT_BYTES or len(stderr) > MAX_OUTPUT_BYTES + ), + } + finally: + try: + os.unlink(script_path) + except FileNotFoundError: + pass + + +class Handler(BaseHTTPRequestHandler): + """Serve health and bounded Python execution requests.""" + + server_version = "HermesChatSandbox/1" + + def log_message(self, format_string: str, *args: object) -> None: + """Keep normal request logs concise and free of request bodies.""" + print(f"sandbox: {self.address_string()} {format_string % args}", flush=True) + + def _json(self, status: int, payload: dict[str, object]) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + """Return a minimal unauthenticated health response.""" + if self.path == "/health": + self._json(200, {"status": "ok"}) + else: + self._json(404, {"error": "not found"}) + + def do_POST(self) -> None: + """Validate and execute a Python request from the matching tenant pod.""" + if self.path != "/v1/execute": + self._json(404, {"error": "not found"}) + return + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + self._json(400, {"error": "invalid content length"}) + return + if length <= 0 or length > MAX_REQUEST_BYTES: + self._json(413, {"error": "request too large"}) + return + try: + payload = json.loads(self.rfile.read(length)) + except (json.JSONDecodeError, UnicodeDecodeError): + self._json(400, {"error": "invalid JSON"}) + return + code = payload.get("code") if isinstance(payload, dict) else None + if not isinstance(code, str) or not code.strip(): + self._json(400, {"error": "code is required"}) + return + if len(code.encode("utf-8")) > MAX_CODE_BYTES: + self._json(413, {"error": "code exceeds 128 KiB"}) + return + self._json(200, _execute(code)) + + +if __name__ == "__main__": + WORKSPACE.mkdir(parents=True, exist_ok=True) + ThreadingHTTPServer((HOST, PORT), Handler).serve_forever() diff --git a/dockerfiles/hermes-public-extract/__init__.py b/dockerfiles/hermes-public-extract/__init__.py new file mode 100644 index 000000000..7196fc1e5 --- /dev/null +++ b/dockerfiles/hermes-public-extract/__init__.py @@ -0,0 +1,8 @@ +"""Bundled public-page extraction provider for isolated Hermes chat.""" + +from plugins.web.public_extract.provider import PublicExtractProvider + + +def register(ctx) -> None: + """Register the credential-free extraction provider.""" + ctx.register_web_search_provider(PublicExtractProvider()) diff --git a/dockerfiles/hermes-public-extract/plugin.yaml b/dockerfiles/hermes-public-extract/plugin.yaml new file mode 100644 index 000000000..fce43b979 --- /dev/null +++ b/dockerfiles/hermes-public-extract/plugin.yaml @@ -0,0 +1,7 @@ +name: web-public-extract +version: 1.0.0 +description: Credential-free extraction for public HTML and text pages. +author: bstein.dev +kind: backend +provides_web_providers: + - public-extract diff --git a/dockerfiles/hermes-public-extract/provider.py b/dockerfiles/hermes-public-extract/provider.py new file mode 100644 index 000000000..1e97576ce --- /dev/null +++ b/dockerfiles/hermes-public-extract/provider.py @@ -0,0 +1,142 @@ +"""Credential-free extraction of bounded public HTML and text pages.""" + +from __future__ import annotations + +from html.parser import HTMLParser +from typing import Any +from urllib.parse import urljoin + +import httpx + +from agent.web_search_provider import WebSearchProvider +from tools.url_safety import is_safe_url +from tools.website_policy import check_website_access + +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_REDIRECTS = 5 + + +class _VisibleTextParser(HTMLParser): + """Collect readable text while discarding scripts, styles, and chrome.""" + + _ignored = {"script", "style", "noscript", "svg", "nav", "footer"} + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._ignore_depth = 0 + self._title_depth = 0 + self.title: list[str] = [] + self.text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + tag = tag.lower() + if tag in self._ignored: + self._ignore_depth += 1 + if tag == "title": + self._title_depth += 1 + if not self._ignore_depth and tag in {"p", "div", "section", "article", "li", "br", "h1", "h2", "h3", "h4"}: + self.text.append("\n") + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + if tag == "title" and self._title_depth: + self._title_depth -= 1 + if tag in self._ignored and self._ignore_depth: + self._ignore_depth -= 1 + if not self._ignore_depth and tag in {"p", "div", "section", "article", "li", "h1", "h2", "h3", "h4"}: + self.text.append("\n") + + def handle_data(self, data: str) -> None: + value = " ".join(data.split()) + if not value: + return + if self._title_depth: + self.title.append(value) + if not self._ignore_depth: + self.text.append(value) + + def readable_text(self) -> str: + """Return normalized paragraphs from collected visible text.""" + lines = [" ".join(line.split()) for line in " ".join(self.text).splitlines()] + return "\n\n".join(line for line in lines if line) + + +def _fetch_public(url: str) -> tuple[str, str, str]: + """Fetch one public URL with redirect, size, MIME, and policy checks.""" + current = url + headers = { + "User-Agent": "HermesPrivateChat/1.0 (+https://chat.hermes.bstein.dev)", + "Accept": "text/html, text/plain;q=0.9, application/xhtml+xml;q=0.8", + } + with httpx.Client(follow_redirects=False, timeout=15.0, headers=headers) as client: + for _ in range(MAX_REDIRECTS + 1): + if not is_safe_url(current): + raise ValueError("URL targets a private or internal network address") + blocked = check_website_access(current) + if blocked: + raise ValueError(blocked.get("message", "URL is blocked by website policy")) + response = client.get(current) + if response.status_code in {301, 302, 303, 307, 308}: + location = response.headers.get("location") + if not location: + raise ValueError("redirect response omitted Location") + current = urljoin(current, location) + continue + response.raise_for_status() + content_type = response.headers.get("content-type", "").lower() + if not any(kind in content_type for kind in ("text/html", "text/plain", "application/xhtml+xml")): + raise ValueError(f"unsupported content type: {content_type or 'unknown'}") + raw = response.content + if len(raw) > MAX_RESPONSE_BYTES: + raise ValueError("page exceeds the 2 MiB extraction limit") + return current, content_type, response.text + raise ValueError("too many redirects") + + +class PublicExtractProvider(WebSearchProvider): + """Extract bounded content directly from public pages without credentials.""" + + @property + def name(self) -> str: + return "public-extract" + + @property + def display_name(self) -> str: + return "Public page extractor" + + def is_available(self) -> bool: + return True + + def supports_search(self) -> bool: + return False + + def supports_extract(self) -> bool: + return True + + def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for url in urls[:20]: + try: + final_url, content_type, body = _fetch_public(url) + if "html" in content_type: + parser = _VisibleTextParser() + parser.feed(body) + title = " ".join(parser.title).strip() + content = parser.readable_text() + else: + title = "" + content = body + results.append( + { + "url": final_url, + "title": title, + "content": content, + "raw_content": content, + "metadata": {"source": "public-extract"}, + } + ) + except Exception as exc: + results.append( + {"url": url, "title": "", "content": "", "error": str(exc)} + ) + return results diff --git a/dockerfiles/hermes-python-sandbox-tool.py b/dockerfiles/hermes-python-sandbox-tool.py new file mode 100644 index 000000000..5689727fc --- /dev/null +++ b/dockerfiles/hermes-python-sandbox-tool.py @@ -0,0 +1,75 @@ +"""Hermes tool for running Python in a separate, credential-free pod.""" + +from __future__ import annotations + +import json +import os +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from tools.registry import registry, tool_error + + +def _sandbox_url() -> str: + """Return the tenant-specific sandbox endpoint injected at startup.""" + return os.environ.get("HERMES_CODE_SANDBOX_URL", "").strip() + + +def _sandbox_available() -> bool: + """Expose the tool only when this tenant has an isolated endpoint.""" + return _sandbox_url().startswith("http://hermes-chat-sandbox-") + + +def execute_python_sandbox(code: str) -> str: + """Execute Python remotely and return the sandbox's structured result.""" + if not isinstance(code, str) or not code.strip(): + return tool_error("Python code is required.") + if len(code.encode("utf-8")) > 128 * 1024: + return tool_error("Python code exceeds the 128 KiB request limit.") + + request = Request( + _sandbox_url(), + data=json.dumps({"code": code}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=55) as response: + payload = response.read(256 * 1024) + return payload.decode("utf-8", errors="replace") + except HTTPError as exc: + detail = exc.read(4096).decode("utf-8", errors="replace") + return tool_error(f"Python sandbox rejected the request: {detail or exc.code}") + except (URLError, TimeoutError, OSError) as exc: + return tool_error(f"Python sandbox is unavailable: {exc}") + + +registry.register( + name="python_sandbox", + toolset="python_sandbox", + schema={ + "name": "python_sandbox", + "description": ( + "Run Python in this user's separate credential-free computation " + "sandbox. Use it for statistics, probability, Monte Carlo simulation, " + "data transforms, and calculations. The sandbox has no Kubernetes, " + "Vault, model-provider credentials, or access to other users. Public " + "research belongs in web_search; pass only the data needed for the " + "calculation and print the result." + ), + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "A self-contained Python 3 program that prints its result.", + } + }, + "required": ["code"], + }, + }, + handler=lambda args, **_: execute_python_sandbox(args.get("code", "")), + check_fn=_sandbox_available, + emoji="🧮", + max_result_size_chars=200_000, +) diff --git a/services/hermes/chat-configmap.yaml b/services/hermes/chat-configmap.yaml index ab88fca7b..0f1eda69c 100644 --- a/services/hermes/chat-configmap.yaml +++ b/services/hermes/chat-configmap.yaml @@ -24,13 +24,21 @@ data: base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 api_key: ollama agent: - api_max_retries: 1 + api_max_retries: 2 + reasoning_effort: high + delegation: + max_concurrent_children: 2 + max_spawn_depth: 1 + web: + backend: ddgs + search_backend: ddgs + extract_backend: public-extract model_catalog: enabled: true ttl_hours: 1 platform_toolsets: - cli: [clarify, file, memory, session_search, skills, todo, web] - api_server: [clarify, file, memory, session_search, skills, todo, web] + cli: [clarify, delegation, file, memory, python_sandbox, session_search, skills, todo, web] + api_server: [clarify, delegation, file, memory, python_sandbox, session_search, skills, todo, web] dashboard: public_url: https://chat.hermes.bstein.dev display: @@ -41,19 +49,30 @@ data: SOUL.md: | You are a high-quality private AI chat assistant. Help the current person with questions, writing, research, planning, and learning. Be direct, - thoughtful, and careful. Use public web research when freshness matters. + thoughtful, and careful. Use public web research when freshness matters, + extract the most relevant primary pages, and cite the sources used. + + Complete complex work instead of stopping after a preflight. For involved + research or analysis, make a short internal plan, delegate independent + research when that improves coverage, use the isolated Python sandbox for + statistics/probability/simulation, check the result, and synthesize one + coherent answer. State assumptions and uncertainty where exact inputs are + unavailable. Never tell the user to enable a tool that is already present. This is a personal sandbox. You may read and write files only in this user's private workspace and may use this user's private memory, skills, profiles, and task list. Never attempt cluster administration, private - service access, terminal execution, credentials, or coordination of Brad's - project agents. The user's conversations and files must never be mixed with - another Keycloak user's state. + service access, credentials, or coordination of Brad's project agents. + Python may run only through the credential-free sandbox tool. The user's + conversations and files must never be mixed with another Keycloak user's + state. AGENTS.md: | # Private Hermes chat This runtime belongs to one authenticated Keycloak identity and one private persistent volume. Provide conversational help with the private workspace, - memory, skills, profiles, task list, session search, and public web tools. + memory, skills, profiles, task list, session search, public web tools, and + the separate per-tenant Python sandbox. Use delegation selectively for + independent research or verification, then present a single final answer. Do not claim access to Kubernetes, Vault, Gitea, Brad's projects, other users, the agent coordinator, or automated triage. diff --git a/services/hermes/chat-sandbox.yaml b/services/hermes/chat-sandbox.yaml new file mode 100644 index 000000000..5d028fab3 --- /dev/null +++ b/services/hermes/chat-sandbox.yaml @@ -0,0 +1,132 @@ +# services/hermes/chat-sandbox.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-chat-sandbox + namespace: hermes + labels: + app: hermes-chat-sandbox +spec: + clusterIP: None + publishNotReadyAddresses: false + selector: + app: hermes-chat-sandbox + ports: + - name: http + port: 9080 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hermes-chat-sandbox + namespace: hermes + labels: + app: hermes-chat-sandbox +spec: + serviceName: hermes-chat-sandbox + replicas: 4 + podManagementPolicy: Parallel + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + selector: + matchLabels: + app: hermes-chat-sandbox + template: + metadata: + labels: + app: hermes-chat-sandbox + annotations: + ai.bstein.dev/role: isolated-user-computation + ai.bstein.dev/isolation: one credential-free sandbox per chat tenant + spec: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 20000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-08, titan-13, titan-14, titan-17, titan-18] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + - weight: 40 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi4] + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: hermes-chat-sandbox + topologyKey: kubernetes.io/hostname + containers: + - name: sandbox + image: registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca + imagePullPolicy: IfNotPresent + ports: + - {name: http, containerPort: 9080, protocol: TCP} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: tmp, mountPath: /tmp} + readinessProbe: + httpGet: {path: /health, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + httpGet: {path: /health, port: http} + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 20000 + runAsGroup: 20000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 1Gi} + volumes: + - name: tmp + emptyDir: + sizeLimit: 256Mi + volumeClaimTemplates: + - metadata: + name: workspace + labels: + app: hermes-chat-sandbox + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 2Gi diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index d30832a83..f4498e009 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -28,7 +28,7 @@ spec: ai.bstein.dev/role: isolated-user-chat ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides - ai.bstein.dev/config-rev: "20260808-webui-telegram" + ai.bstein.dev/config-rev: "20260808-quality-sandbox" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-chat vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -158,10 +158,14 @@ spec: limits: {cpu: 100m, memory: 128Mi} containers: - name: hermes - image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + image: registry.bstein.dev/bstein/hermes-agent@sha256:2f3b8299e0b72c94e2daf1d2f313256b5bae6ed7f643d38d1ddca13ab397a85c imagePullPolicy: IfNotPresent - command: [/opt/hermes/.venv/bin/hermes] - args: [gateway, run] + command: [/bin/sh, -ec] + args: + - | + ordinal="${HOSTNAME##*-}" + export HERMES_CODE_SANDBOX_URL="http://hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox.hermes.svc.cluster.local:9080/v1/execute" + exec /opt/hermes/.venv/bin/hermes gateway run ports: - {name: api, containerPort: 8642, protocol: TCP} env: diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 37e54d65f..c8aec9732 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -21,6 +21,7 @@ resources: - deployment.yaml - agent-deployment.yaml - chat-statefulset.yaml + - chat-sandbox.yaml - chat-router.yaml - service.yaml - oauth2-proxy.yaml diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index bb388cf23..6eed18bb0 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -130,6 +130,99 @@ spec: --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy +metadata: + name: hermes-chat-sandbox-deny + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-sandbox + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-sandbox-tenant-0 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-sandbox + apps.kubernetes.io/pod-index: "0" + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-chat-tenant + apps.kubernetes.io/pod-index: "0" + ports: + - {protocol: TCP, port: 9080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-sandbox-tenant-1 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-sandbox + apps.kubernetes.io/pod-index: "1" + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-chat-tenant + apps.kubernetes.io/pod-index: "1" + ports: + - {protocol: TCP, port: 9080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-sandbox-tenant-2 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-sandbox + apps.kubernetes.io/pod-index: "2" + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-chat-tenant + apps.kubernetes.io/pod-index: "2" + ports: + - {protocol: TCP, port: 9080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-sandbox-tenant-3 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-sandbox + apps.kubernetes.io/pod-index: "3" + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-chat-tenant + apps.kubernetes.io/pod-index: "3" + ports: + - {protocol: TCP, port: 9080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy metadata: name: hermes-chat-tenant-isolation namespace: hermes @@ -181,6 +274,12 @@ spec: app: hermes-model-gate ports: - {protocol: TCP, port: 8080} + - to: + - podSelector: + matchLabels: + app: hermes-chat-sandbox + ports: + - {protocol: TCP, port: 9080} - to: - ipBlock: cidr: 0.0.0.0/0 diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py new file mode 100644 index 000000000..893d94e9c --- /dev/null +++ b/testing/tests/test_hermes_chat_quality.py @@ -0,0 +1,100 @@ +"""Contracts for isolated high-quality Hermes chat capabilities.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" + + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + + +def test_chat_config_enables_real_research_compute_and_delegation(): + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + + assert config["agent"]["reasoning_effort"] == "high" + assert config["web"] == { + "backend": "ddgs", + "search_backend": "ddgs", + "extract_backend": "public-extract", + } + assert config["delegation"]["max_concurrent_children"] == 2 + for platform in ("cli", "api_server"): + toolsets = config["platform_toolsets"][platform] + assert "delegation" in toolsets + assert "python_sandbox" in toolsets + assert "web" in toolsets + assert "terminal" not in toolsets + assert "code_execution" not in toolsets + + +def test_sandbox_has_no_credentials_token_or_egress(): + sandbox_docs = _documents(HERMES / "chat-sandbox.yaml") + statefulset = next(doc for doc in sandbox_docs if doc["kind"] == "StatefulSet") + pod_spec = statefulset["spec"]["template"]["spec"] + container = pod_spec["containers"][0] + + assert pod_spec["automountServiceAccountToken"] is False + assert container["securityContext"]["readOnlyRootFilesystem"] is True + assert container["securityContext"]["runAsNonRoot"] is True + assert not container.get("env") + assert {mount["mountPath"] for mount in container["volumeMounts"]} == { + "/tmp", + "/workspace", + } + + policies = _documents(HERMES / "networkpolicy.yaml") + deny = next( + doc + for doc in policies + if doc["kind"] == "NetworkPolicy" + and doc["metadata"]["name"] == "hermes-chat-sandbox-deny" + ) + assert deny["spec"]["policyTypes"] == ["Ingress", "Egress"] + assert deny["spec"]["egress"] == [] + for ordinal in range(4): + policy = next( + doc + for doc in policies + if doc["kind"] == "NetworkPolicy" + and doc["metadata"]["name"] == f"hermes-chat-sandbox-tenant-{ordinal}" + ) + assert policy["spec"]["podSelector"]["matchLabels"][ + "apps.kubernetes.io/pod-index" + ] == str(ordinal) + source = policy["spec"]["ingress"][0]["from"][0]["podSelector"][ + "matchLabels" + ] + assert source["apps.kubernetes.io/pod-index"] == str(ordinal) + + +def test_gateway_image_honors_ui_model_and_caps_reasoning(): + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() + assert "_resolve_request_route" in dockerfile + assert 'allowed_providers = {"openai-codex", "anthropic"}' in dockerfile + assert 'reasoning_effort=body.get("reasoning_effort")' in dockerfile + assert 'reasoning_config = {"enabled": True, "effort": "xhigh"}' in dockerfile + assert "ddgs==9.14.4" in dockerfile + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == ""