From 878cfb1d0083f72341d025cd5314f988b0d0aecd Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 05:20:18 -0300 Subject: [PATCH] hermes: add automatic hosted and local image routes --- dockerfiles/Dockerfile.hermes-local-image | 19 ++ dockerfiles/hermes-local-image-server.py | 297 ++++++++++++++++++ services/ai-llm/deployment.yaml | 21 +- services/hermes/agent-configmap.yaml | 2 +- services/hermes/agent-deployment.yaml | 10 +- services/hermes/chat-configmap.yaml | 20 +- services/hermes/chat-statefulset.yaml | 7 +- services/hermes/configmap.yaml | 4 - services/hermes/deployment.yaml | 4 +- services/hermes/image-policy-configmap.yaml | 11 + services/hermes/kustomization.yaml | 4 +- services/hermes/local-image-deployment.yaml | 106 +++++++ services/hermes/local-image-rbac.yaml | 36 +++ services/hermes/model-gate-configmap.yaml | 167 ++++++---- services/hermes/model-gate-deployment.yaml | 32 +- services/hermes/model-gate-rbac.yaml | 6 +- services/hermes/networkpolicy.yaml | 36 ++- services/hermes/ollama-deployment.yaml | 120 ------- .../hermes/plugins/auto-router/__init__.py | 190 ++++++++++- .../hermes/plugins/auto-router/plugin.yaml | 2 +- .../plugins/image-gen-broker/__init__.py | 39 ++- services/hermes/pvc.yaml | 4 +- .../hermes/scripts/hermes_model_routing.py | 5 +- services/hermes/scripts/image_broker.py | 187 +++++++++-- services/hermes/service.yaml | 10 +- .../maintenance/apps/ariadne-deployment.yaml | 6 +- testing/tests/test_hermes_auto_router.py | 87 +++++ testing/tests/test_hermes_chat_quality.py | 149 ++++++++- testing/tests/test_hermes_model_gate.py | 25 ++ 29 files changed, 1315 insertions(+), 291 deletions(-) create mode 100644 dockerfiles/Dockerfile.hermes-local-image create mode 100644 dockerfiles/hermes-local-image-server.py create mode 100644 services/hermes/image-policy-configmap.yaml create mode 100644 services/hermes/local-image-deployment.yaml create mode 100644 services/hermes/local-image-rbac.yaml delete mode 100644 services/hermes/ollama-deployment.yaml diff --git a/dockerfiles/Dockerfile.hermes-local-image b/dockerfiles/Dockerfile.hermes-local-image new file mode 100644 index 000000000..da90cf370 --- /dev/null +++ b/dockerfiles/Dockerfile.hermes-local-image @@ -0,0 +1,19 @@ +# dockerfiles/Dockerfile.hermes-local-image +FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime@sha256:77f17f843507062875ce8be2a6f76aa6aa3df7f9ef1e31d9d7432f4b0f563dee + +ARG DIFFUSERS_COMMIT=90c0ffdc045902a3667d473d2fbfc03e8716dba9 +RUN python -m pip install --no-cache-dir \ + "https://github.com/huggingface/diffusers/archive/${DIFFUSERS_COMMIT}.tar.gz" \ + "transformers==5.15.0" \ + "accelerate==1.14.0" \ + "safetensors==0.8.0" \ + "huggingface-hub==1.27.0" \ + "Pillow==12.3.0" +RUN python -c "from diffusers import Flux2KleinPipeline; print(Flux2KleinPipeline.__name__)" + +COPY dockerfiles/hermes-local-image-server.py /opt/hermes/local_image_server.py +ENV PYTHONUNBUFFERED=1 \ + HF_HOME=/models/huggingface \ + HF_HUB_DISABLE_TELEMETRY=1 +EXPOSE 9004 +ENTRYPOINT ["python", "/opt/hermes/local_image_server.py"] diff --git a/dockerfiles/hermes-local-image-server.py b/dockerfiles/hermes-local-image-server.py new file mode 100644 index 000000000..a69c7cbfd --- /dev/null +++ b/dockerfiles/hermes-local-image-server.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Lease-aware FLUX image generation for the shared titan-24 GPU.""" + +from __future__ import annotations + +import base64 +import gc +import io +import json +import os +import ssl +import threading +from datetime import UTC, datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +HOST = os.environ.get("HERMES_LOCAL_IMAGE_HOST", "0.0.0.0") +PORT = int(os.environ.get("HERMES_LOCAL_IMAGE_PORT", "9004")) +MODEL_ID = os.environ.get( + "HERMES_LOCAL_IMAGE_MODEL", "black-forest-labs/FLUX.2-klein-4B" +) +MODEL_REVISION = os.environ.get( + "HERMES_LOCAL_IMAGE_REVISION", "e7b7dc27f91deacad38e78976d1f2b499d76a294" +) +MODEL_CACHE = os.environ.get("HF_HOME", "/models/huggingface") +LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes") +LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner") +LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes") +LEASE_IMAGE_OWNER = os.environ.get("LEASE_IMAGE_OWNER", "hermes-image") +MAX_BODY_BYTES = int(os.environ.get("HERMES_LOCAL_IMAGE_MAX_BODY", str(96 << 20))) +QUEUE_TIMEOUT_SECONDS = float(os.environ.get("HERMES_LOCAL_IMAGE_QUEUE_TIMEOUT", "1200")) +API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") +API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") +TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") +CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") +LEASE_URL = ( + f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/" + f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}" +) +ASPECTS = { + "square": (1024, 1024), + "landscape": (768, 1280), + "portrait": (1280, 768), +} + +_generation_lock = threading.Lock() +_state_lock = threading.Lock() +_state: dict[str, Any] = { + "busy": False, + "phase": "idle", + "last_error": None, + "last_model": None, +} + + +def _set_state(**values: Any) -> None: + """Update health state without exposing prompts or image content.""" + with _state_lock: + _state.update(values) + + +def _state_snapshot() -> dict[str, Any]: + """Return a stable copy for health responses.""" + with _state_lock: + return dict(_state) + + +def _kube_request(method: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """Call the Lease API using the pod service account.""" + token = TOKEN_PATH.read_text(encoding="utf-8").strip() + raw = None if body is None else json.dumps(body).encode("utf-8") + headers = {"Authorization": f"Bearer {token}"} + if raw is not None: + headers["Content-Type"] = "application/merge-patch+json" + request = Request(LEASE_URL, data=raw, headers=headers, method=method) + context = ssl.create_default_context(cafile=str(CA_PATH)) + with urlopen(request, timeout=5, context=context) as response: + return json.load(response) + + +def _lease_owner() -> str: + """Read the current GPU owner, failing closed on API errors.""" + payload = _kube_request("GET") + return str((payload.get("spec") or {}).get("holderIdentity") or "unavailable") + + +def _change_owner(expected: str, desired: str) -> bool: + """Change owner only if the resource version and holder still match.""" + current = _kube_request("GET") + if str((current.get("spec") or {}).get("holderIdentity")) != expected: + return False + resource_version = str((current.get("metadata") or {}).get("resourceVersion") or "") + if not resource_version: + return False + patch = { + "metadata": {"resourceVersion": resource_version}, + "spec": { + "holderIdentity": desired, + "renewTime": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + }, + } + try: + _kube_request("PATCH", patch) + except HTTPError as exc: + if exc.code == 409: + return False + raise + return True + + +def _recover_stale_image_lease() -> None: + """Release this deployment's lease after a prior renderer crash.""" + try: + if _lease_owner() == LEASE_IMAGE_OWNER: + _change_owner(LEASE_IMAGE_OWNER, LEASE_IDLE_OWNER) + except Exception as exc: + _set_state(last_error=f"lease recovery failed: {type(exc).__name__}: {exc}") + + +def _decode_image(value: str): + """Decode a bounded image data URL for FLUX editing.""" + from PIL import Image + + if not value.startswith("data:image/") or ";base64," not in value: + raise ValueError("local reference images must be data URLs") + raw = base64.b64decode(value.split(",", 1)[1], validate=True) + if not raw or len(raw) > 25 << 20: + raise ValueError("reference image must be between 1 byte and 25 MiB") + return Image.open(io.BytesIO(raw)).convert("RGB") + + +def _render(payload: dict[str, Any]) -> dict[str, Any]: + """Load FLUX for one request, render, then fully release its GPU memory.""" + import torch + from diffusers import Flux2KleinPipeline + + prompt = str(payload.get("prompt") or "").strip() + if not prompt: + raise ValueError("prompt is required") + aspect = str(payload.get("aspect_ratio") or "landscape").lower() + if aspect not in ASPECTS: + raise ValueError("aspect_ratio must be landscape, square, or portrait") + height, width = ASPECTS[aspect] + inputs = [] + if payload.get("image_url"): + inputs.append(_decode_image(str(payload["image_url"]))) + for item in payload.get("reference_image_urls") or []: + inputs.append(_decode_image(str(item))) + inputs = inputs[:4] + + _set_state(phase="loading-flux", last_model=MODEL_ID) + pipe = None + try: + pipe = Flux2KleinPipeline.from_pretrained( + MODEL_ID, + revision=MODEL_REVISION, + torch_dtype=torch.bfloat16, + cache_dir=MODEL_CACHE, + ) + pipe.enable_model_cpu_offload() + _set_state(phase="rendering") + arguments: dict[str, Any] = { + "prompt": prompt, + "height": height, + "width": width, + "guidance_scale": 1.0, + "num_inference_steps": 4, + } + if inputs: + arguments["image"] = inputs + image = pipe(**arguments).images[0] + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return { + "success": True, + "image_b64": base64.b64encode(output.getvalue()).decode("ascii"), + "mime_type": "image/png", + "model": "flux-2-klein-4b-local", + "quality": "local-high", + "size": f"{width}x{height}", + "input_image_count": len(inputs), + "route": "local", + } + finally: + del pipe + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + +def _generate(payload: dict[str, Any]) -> dict[str, Any]: + """Serialize renders and perform an exclusive lease-backed GPU handoff.""" + acquired = _generation_lock.acquire(timeout=QUEUE_TIMEOUT_SECONDS) + if not acquired: + raise TimeoutError("local image queue is busy; retry shortly") + claimed = False + _set_state(busy=True, phase="claiming-gpu", last_error=None) + try: + claimed = _change_owner(LEASE_IDLE_OWNER, LEASE_IMAGE_OWNER) + if not claimed: + owner = _lease_owner() + raise RuntimeError(f"local image GPU unavailable while titan-24 owner is {owner}") + result = _render(payload) + _set_state(phase="releasing-flux") + if _lease_owner() == LEASE_IMAGE_OWNER: + if not _change_owner(LEASE_IMAGE_OWNER, LEASE_IDLE_OWNER): + raise RuntimeError("GPU ownership changed while releasing the image lane") + return result + except Exception as exc: + _set_state(last_error=f"{type(exc).__name__}: {exc}") + raise + finally: + # If Wolf has moved the lease to wolf-draining, never overwrite that + # transition. Ariadne waits for busy=false before using the GPU. + if claimed: + try: + if _lease_owner() == LEASE_IMAGE_OWNER: + _change_owner(LEASE_IMAGE_OWNER, LEASE_IDLE_OWNER) + except Exception as exc: + _set_state(last_error=f"restore failed: {type(exc).__name__}: {exc}") + _set_state(busy=False, phase="idle") + _generation_lock.release() + + +class Handler(BaseHTTPRequestHandler): + """Internal JSON interface used only by the owner image broker.""" + + server_version = "HermesLocalImage/1" + + def _json(self, status: int, value: dict[str, Any]) -> None: + body = json.dumps(value, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + if self.path != "/health": + self._json(404, {"success": False, "error": "not found"}) + return + state = _state_snapshot() + try: + owner = _lease_owner() + except Exception: + owner = "unavailable" + self._json( + 200, + { + "success": True, + "available": owner == LEASE_IDLE_OWNER and not state["busy"], + "gpu_owner": owner, + "model": "flux-2-klein-4b-local", + **state, + }, + ) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/v1/images/generations": + self._json(404, {"success": False, "error": "not found"}) + return + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + if length <= 0 or length > MAX_BODY_BYTES: + self._json(413, {"success": False, "error": "invalid request size"}) + return + try: + payload = json.loads(self.rfile.read(length)) + if not isinstance(payload, dict): + raise ValueError("JSON object required") + result = _generate(payload) + self._json(200, result) + except ValueError as exc: + self._json(400, {"success": False, "error": str(exc)}) + except TimeoutError as exc: + self._json(503, {"success": False, "error": str(exc)}) + except Exception as exc: + self._json( + 503, + {"success": False, "error": f"local image generation failed: {type(exc).__name__}: {exc}"}, + ) + + def log_message(self, format_string: str, *args: Any) -> None: + print(f"local-image {self.address_string()} {format_string % args}", flush=True) + + +if __name__ == "__main__": + _recover_stale_image_lease() + ThreadingHTTPServer((HOST, PORT), Handler).serve_forever() diff --git a/services/ai-llm/deployment.yaml b/services/ai-llm/deployment.yaml index 06db8282b..d31334268 100644 --- a/services/ai-llm/deployment.yaml +++ b/services/ai-llm/deployment.yaml @@ -22,7 +22,7 @@ spec: annotations: ai.bstein.dev/model: qwen2.5:3b-instruct-q4_0,qwen2.5:14b-instruct-q4_0 ai.bstein.dev/gpu: titan-20 shared routing GPU - ai.bstein.dev/restartedAt: "2026-01-26T12:00:00Z" + ai.bstein.dev/restartedAt: "2026-08-11T08:00:00Z" spec: affinity: nodeAffinity: @@ -94,9 +94,22 @@ spec: - name: OLLAMA_FAST_MODEL value: qwen2.5:3b-instruct-q4_0 - name: OLLAMA_CONTEXT_LENGTH - value: "512" + value: "8192" - name: OLLAMA_KEEP_ALIVE - value: 6h + value: "-1" + # The Xavier has 16 GiB of unified memory. Qwen 3B + Qwen 14B + # weights alone use about 10.3 GB, so concurrent residency leaves + # no safe room for KV cache, CUDA, kubelet, or the OS. Serialize + # model residency; the router re-warms the classifier after a + # local 14B answer. + - name: OLLAMA_MAX_LOADED_MODELS + value: "1" + - name: OLLAMA_NUM_PARALLEL + value: "1" + - name: OLLAMA_FLASH_ATTENTION + value: "1" + - name: OLLAMA_KV_CACHE_TYPE + value: q8_0 - name: OLLAMA_MODELS value: /root/.ollama - name: NVIDIA_VISIBLE_DEVICES @@ -114,7 +127,7 @@ spec: pid="$!" trap 'kill -TERM "$pid"; wait "$pid"' TERM INT sleep 6 - timeout 180s ollama run "${OLLAMA_FAST_MODEL}" --keepalive 24h "reply with just pong" >/tmp/ollama-fast-warm.log 2>&1 + timeout 180s ollama run "${OLLAMA_FAST_MODEL}" --keepalive -1 "reply with just pong" >/tmp/ollama-fast-warm.log 2>&1 touch /tmp/ollama-fast-ready wait "$pid" volumeMounts: diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 5c6f4a3f3..e16119310 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -20,7 +20,7 @@ data: - provider: anthropic model: claude-sonnet-5 - provider: custom - model: gpt-oss:20b + model: qwen2.5:14b-instruct-q4_0 base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 api_key: ollama diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 7367a9e70..63327ebc2 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260811-redis-sessions" + ai.bstein.dev/config-rev: "20260811-image-routing" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -707,8 +707,10 @@ spec: - {name: HOME, value: /opt/data/home} - {name: CODEX_HOME, value: /opt/data/home/.codex} - {name: PYTHONPATH, value: /opt/hermes} - - {name: HERMES_IMAGE_BROKER_DEFAULT_MODEL, value: gpt-image-2-high} + - {name: HERMES_IMAGE_BROKER_DEFAULT_MODEL, value: atlas-image-auto-high} - {name: HERMES_IMAGE_BROKER_LISTEN_PORT, value: "9002"} + - {name: HERMES_LOCAL_IMAGE_URL, value: http://hermes-local-image.hermes.svc.cluster.local:9004} + - {name: HERMES_IMAGE_POLICY_PATH, value: /etc/hermes-image-policy/policy.json} readinessProbe: tcpSocket: {port: image-broker} initialDelaySeconds: 5 @@ -732,6 +734,7 @@ spec: - {name: provider-auth, mountPath: /shared-auth} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/auxiliary_client.py, subPath: auxiliary_client.py} + - {name: image-policy, mountPath: /etc/hermes-image-policy, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 50m, memory: 128Mi} @@ -809,6 +812,9 @@ spec: - name: auto-router-plugin configMap: name: hermes-auto-router-plugin + - name: image-policy + configMap: + name: hermes-image-policy - name: tmp emptyDir: sizeLimit: 4Gi diff --git a/services/hermes/chat-configmap.yaml b/services/hermes/chat-configmap.yaml index e0351d9b2..bcd8adf68 100644 --- a/services/hermes/chat-configmap.yaml +++ b/services/hermes/chat-configmap.yaml @@ -24,10 +24,6 @@ data: model: claude-sonnet-5 - provider: custom model: qwen2.5:14b-instruct-q4_0 - base_url: http://ollama.ai.svc.cluster.local:11434/v1 - api_key: ollama - - provider: custom - model: gpt-oss:20b base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 api_key: ollama agent: @@ -50,10 +46,11 @@ data: language: auto image_gen: provider: atlas-broker - model: gpt-image-2-high + model: atlas-image-auto-high plugins: enabled: - atlas-broker + - auto-router model_catalog: enabled: true ttl_hours: 1 @@ -120,11 +117,18 @@ data: transport services only; they do not select or replace the answering model. When a user asks to create or edit an image, use the image generation tool. - Default to its highest-quality GPT Image tier, honor requested aspect ratio, + Use `atlas-image-auto-high` unless the user selects a route: requests that + say local, private, on my hardware, or FLUX use `flux-2-klein-4b-local`; + requests that say OpenAI, hosted, GPT Image, or highest hosted quality use + `gpt-image-2-high`. AUTO tries GPT Image 2 High first and falls back to local + FLUX when the hosted route fails or refuses. Honor requested aspect ratio and use uploaded images as references when provided. Return the generated image inline so the WebUI offers its normal preview and download controls. - Do not substitute Python drawing, SVG, diagrams, or placeholder artwork for - a requested generative image. + The local FLUX route is already provisioned: never tell a user to install + Diffusers, download a checkpoint, or write a Python generation script. If + local rendering is unavailable because Wolf owns titan-24, say so plainly + and offer AUTO or OpenAI. Do not substitute Python drawing, SVG, diagrams, + or placeholder artwork for a requested generative image. AGENTS.md: | # Private Hermes chat diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index cca28afee..c7d7e17be 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -28,7 +28,7 @@ spec: ai.bstein.dev/role: isolated-user-chat ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides - ai.bstein.dev/config-rev: "20260811-codex-broker" + ai.bstein.dev/config-rev: "20260811-image-routing" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-chat vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -207,6 +207,7 @@ spec: - {name: API_SERVER_PORT, value: "8642"} - {name: API_SERVER_CORS_ORIGINS, value: https://chat.hermes.bstein.dev} - {name: HERMES_IMAGE_BROKER_URL, value: http://hermes-image-broker.hermes.svc.cluster.local:9002} + - {name: HERMES_AUTO_ROUTER_CHAT_MODE, value: "1"} volumeMounts: - {name: home, mountPath: /opt/data} - {name: workspace, mountPath: /opt/data/workspace} @@ -216,6 +217,7 @@ spec: - {name: provider-auth, mountPath: /shared-auth} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: image-plugin, mountPath: /opt/hermes/plugins/image_gen/atlas-broker, readOnly: true} + - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} readinessProbe: tcpSocket: {port: api} initialDelaySeconds: 30 @@ -309,6 +311,9 @@ spec: defaultMode: 0555 - name: auth-patch emptyDir: {} + - name: auto-router-plugin + configMap: + name: hermes-auto-router-plugin - name: image-plugin configMap: name: hermes-chat-image-plugin diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index 1dc290b51..7e9d7d481 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -18,10 +18,6 @@ data: model: gpt-5.6-terra - provider: custom model: qwen2.5:14b-instruct-q4_0 - base_url: http://ollama.ai.svc.cluster.local:11434/v1 - api_key: ollama - - provider: custom - model: gpt-oss:20b base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 api_key: ollama diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 6b86fd8ee..1564b265e 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -21,10 +21,10 @@ spec: app: hermes annotations: ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation - ai.bstein.dev/model: anthropic/claude-opus-5, falling back to openai-codex/gpt-5.6-terra then local gpt-oss:20b + ai.bstein.dev/model: anthropic/claude-opus-5, falling back to openai-codex/gpt-5.6-terra then titan-20 Qwen 14B ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: titan-21 preferred, Jetson preferred, arm64 fallback - ai.bstein.dev/config-rev: "20260810-noise-gated-webui" + ai.bstein.dev/config-rev: "20260811-titan20-local-fallback" # The Anthropic credential comes from Vault rather than a manually # created Secret. The role is declared in # services/vault/scripts/vault_k8s_auth_configure.sh and bound to diff --git a/services/hermes/image-policy-configmap.yaml b/services/hermes/image-policy-configmap.yaml new file mode 100644 index 000000000..04376a5d6 --- /dev/null +++ b/services/hermes/image-policy-configmap.yaml @@ -0,0 +1,11 @@ +# services/hermes/image-policy-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-image-policy + namespace: hermes +data: + policy.json: | + { + "additional_blocked_phrases": [] + } diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index ed9a44dfb..306e4f7f8 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -14,12 +14,14 @@ resources: - chat-pvcs.yaml - oauth-session-store.yaml - model-gate-rbac.yaml + - local-image-rbac.yaml - ariadne-handoff-rbac.yaml - model-gate-state.yaml - model-gate-configmap.yaml - model-gate-deployment.yaml + - image-policy-configmap.yaml - networkpolicy.yaml - - ollama-deployment.yaml + - local-image-deployment.yaml - deployment.yaml - agent-deployment.yaml - voice-deployment.yaml diff --git a/services/hermes/local-image-deployment.yaml b/services/hermes/local-image-deployment.yaml new file mode 100644 index 000000000..a1d1f2c94 --- /dev/null +++ b/services/hermes/local-image-deployment.yaml @@ -0,0 +1,106 @@ +# services/hermes/local-image-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-local-image + namespace: hermes + labels: + app: hermes-local-image +spec: + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: hermes-local-image + template: + metadata: + labels: + app: hermes-local-image + annotations: + ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B + ai.bstein.dev/gpu: titan-24 lease-shared image and Wolf lane + ai.bstein.dev/config-rev: "20260811-flux2-klein-image-only" + spec: + serviceAccountName: hermes-gpu-runtime + runtimeClassName: nvidia + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - titan-24 + volumes: + - name: models + persistentVolumeClaim: + claimName: hermes-image-models + - name: local-image-tmp + emptyDir: + sizeLimit: 2Gi + containers: + - name: local-image + image: registry.bstein.dev/bstein/hermes-local-image@sha256:2bf0e5b19ef80cf0cfe7b3d9a4b672df101a3215634a6cede85ff850ada7a28a + imagePullPolicy: IfNotPresent + ports: + - name: local-image + containerPort: 9004 + env: + - name: HERMES_LOCAL_IMAGE_MODEL + value: black-forest-labs/FLUX.2-klein-4B + - name: HERMES_LOCAL_IMAGE_REVISION + value: e7b7dc27f91deacad38e78976d1f2b499d76a294 + - name: LEASE_NAMESPACE + value: hermes + - name: LEASE_NAME + value: titan-24-gpu-owner + - name: LEASE_IDLE_OWNER + value: hermes + - name: LEASE_IMAGE_OWNER + value: hermes-image + - name: HF_HOME + value: /models/huggingface + - name: HOME + value: /tmp + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: compute,utility + volumeMounts: + - name: models + mountPath: /models + - name: local-image-tmp + mountPath: /tmp + readinessProbe: + httpGet: + path: /health + port: local-image + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: local-image + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: "4" + memory: 12Gi + nvidia.com/gpu.shared: 1 + limits: + cpu: "12" + memory: 28Gi + nvidia.com/gpu.shared: 1 diff --git a/services/hermes/local-image-rbac.yaml b/services/hermes/local-image-rbac.yaml new file mode 100644 index 000000000..ea5295172 --- /dev/null +++ b/services/hermes/local-image-rbac.yaml @@ -0,0 +1,36 @@ +# services/hermes/local-image-rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-gpu-runtime + namespace: hermes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hermes-gpu-runtime + namespace: hermes +rules: + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + resourceNames: + - titan-24-gpu-owner + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hermes-gpu-runtime + namespace: hermes +subjects: + - kind: ServiceAccount + name: hermes-gpu-runtime + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hermes-gpu-runtime diff --git a/services/hermes/model-gate-configmap.yaml b/services/hermes/model-gate-configmap.yaml index 4dfde98fb..880846c8c 100644 --- a/services/hermes/model-gate-configmap.yaml +++ b/services/hermes/model-gate-configmap.yaml @@ -7,7 +7,7 @@ metadata: data: model_gate.py: | #!/usr/bin/env python3 - """Fail-closed proxy that admits local inference only while Hermes owns titan-24.""" + """Normalize Jetson text requests and coordinate the titan-24 image handoff.""" from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json @@ -22,44 +22,33 @@ data: LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0") LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080")) - UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/") - LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes") - LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner") - CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1")) - API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") - API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") - TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") - CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") - LEASE_URL = ( - f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/" - f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}" - ) - - _cache_lock = threading.Lock() - _cached_owner = "unavailable" - _cached_at = 0.0 + HANDOFF_PORT = int(os.environ.get("HANDOFF_PORT", "8081")) + UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://ollama.ai.svc.cluster.local:11434").rstrip("/") + LOCAL_IMAGE_URL = os.environ.get("LOCAL_IMAGE_URL", "http://hermes-local-image.hermes.svc.cluster.local:9004").rstrip("/") + HANDOFF_TIMEOUT_SEC = float(os.environ.get("HANDOFF_TIMEOUT_SEC", "1200")) + IMAGE_NAMESPACE = os.environ.get("IMAGE_NAMESPACE", "hermes") + IMAGE_DEPLOYMENT = os.environ.get("IMAGE_DEPLOYMENT", "hermes-local-image") + KUBE_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") + KUBE_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") + KUBE_TOKEN = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") + KUBE_CA = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + _handoff_lock = threading.Lock() + _image_lane_warm = True - def _lease_owner() -> str: - """Return the current owner, failing closed when Kubernetes is unavailable.""" + def _image_deployment_ready() -> bool: + """Fail closed on transient network loss, but do not wait on a dead pod.""" - global _cached_at, _cached_owner - now = time.monotonic() - with _cache_lock: - if now - _cached_at < CACHE_TTL_SEC: - return _cached_owner - try: - token = TOKEN_PATH.read_text(encoding="utf-8").strip() - request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"}) - context = ssl.create_default_context(cafile=str(CA_PATH)) - with urlopen(request, timeout=3, context=context) as response: - payload = json.load(response) - owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip() - except Exception: - owner = "unavailable" - _cached_owner = owner - _cached_at = now - return owner + token = KUBE_TOKEN.read_text(encoding="utf-8").strip() + url = ( + f"https://{KUBE_HOST}:{KUBE_PORT}/apis/apps/v1/namespaces/" + f"{IMAGE_NAMESPACE}/deployments/{IMAGE_DEPLOYMENT}" + ) + request = Request(url, headers={"Authorization": f"Bearer {token}"}) + context = ssl.create_default_context(cafile=str(KUBE_CA)) + with urlopen(request, timeout=5, context=context) as response: + payload = json.load(response) + return int((payload.get("status") or {}).get("readyReplicas") or 0) > 0 def _normalize_reasoning(body: bytes | None) -> bytes | None: @@ -90,8 +79,32 @@ data: return json.dumps(payload, separators=(",", ":")).encode("utf-8") + def _wait_for_image_idle() -> tuple[bool, str]: + """Keep Wolf handoff blocked until an in-flight FLUX render is released.""" + + deadline = time.monotonic() + HANDOFF_TIMEOUT_SEC + last_error = "local image service did not report idle" + while time.monotonic() < deadline: + try: + with urlopen(f"{LOCAL_IMAGE_URL}/health", timeout=5) as response: + payload = json.load(response) + if not bool(payload.get("busy")): + return True, "" + last_error = f"local image renderer is {payload.get('phase', 'busy')}" + except Exception as exc: + last_error = f"local image health unavailable: {exc}" + try: + if not _image_deployment_ready(): + # A stopped/crashed process has no CUDA context left to drain. + return True, "" + except Exception as kube_exc: + last_error += f"; deployment state unavailable: {kube_exc}" + time.sleep(1) + return False, last_error + + class Handler(BaseHTTPRequestHandler): - """Proxy local model traffic while exposing health and ownership status.""" + """Proxy the non-preemptible titan-20 text fallback.""" protocol_version = "HTTP/1.1" @@ -104,26 +117,7 @@ data: self.end_headers() self.wfile.write(body) - def _local_allowed(self) -> tuple[bool, str]: - owner = _lease_owner() - return owner == "hermes", owner - def _proxy(self) -> None: - allowed, owner = self._local_allowed() - if not allowed: - self._json( - 503, - { - "error": { - "message": f"local GPU inference unavailable while titan-24 owner is {owner}", - "type": "server_error", - }, - "gpu_owner": owner, - "fallback_required": True, - }, - ) - return - length = int(self.headers.get("Content-Length", "0") or 0) body = self.rfile.read(length) if length else None body = _normalize_reasoning(body) @@ -164,8 +158,7 @@ data: self._json(200, {"status": "ok"}) return if self.path == "/gate/status": - allowed, owner = self._local_allowed() - self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed}) + self._json(200, {"local_inference_allowed": True, "node": "titan-20"}) return self._proxy() @@ -176,5 +169,63 @@ data: print(f"model-gate {self.address_string()} {format_string % args}", flush=True) + class HandoffHandler(BaseHTTPRequestHandler): + """Ollama-compatible adapter for Ariadne's image/Wolf handoff contract.""" + + protocol_version = "HTTP/1.1" + + def _json(self, status: int, payload: dict) -> None: + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _proxy(self) -> None: + global _image_lane_warm + if self.command == "GET" and self.path == "/api/ps": + models = [{"name": "flux-2-klein-4b-local"}] if _image_lane_warm else [] + self._json(200, {"models": models}) + return + if self.command != "POST" or self.path != "/api/generate": + self._json(404, {"error": "unsupported handoff operation"}) + return + length = int(self.headers.get("Content-Length", "0") or 0) + try: + payload = json.loads(self.rfile.read(length) or b"{}") + except (ValueError, json.JSONDecodeError): + self._json(400, {"error": "invalid JSON"}) + return + keep_alive = payload.get("keep_alive") + with _handoff_lock: + idle, error = _wait_for_image_idle() + if not idle: + self._json(503, {"error": error, "gpu_handoff_blocked": True}) + return + if keep_alive == 0: + _image_lane_warm = False + self._json(200, {"response": "", "done": True}) + return + _image_lane_warm = True + response = "READY" if str(payload.get("prompt") or "").strip() else "" + self._json(200, {"response": response, "done": True}) + + def do_GET(self) -> None: + if self.path == "/healthz": + self._json(200, {"status": "ok"}) + return + self._proxy() + + def do_POST(self) -> None: + self._proxy() + + def log_message(self, format_string: str, *args) -> None: + print(f"gpu-handoff {self.address_string()} {format_string % args}", flush=True) + + if __name__ == "__main__": + handoff = ThreadingHTTPServer((LISTEN_HOST, HANDOFF_PORT), HandoffHandler) + threading.Thread(target=handoff.serve_forever, name="gpu-handoff", daemon=True).start() ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever() diff --git a/services/hermes/model-gate-deployment.yaml b/services/hermes/model-gate-deployment.yaml index 69f79a153..f47c611ae 100644 --- a/services/hermes/model-gate-deployment.yaml +++ b/services/hermes/model-gate-deployment.yaml @@ -15,7 +15,7 @@ spec: template: metadata: annotations: - ai.bstein.dev/config-rev: "20260811-reasoning-clamp" + ai.bstein.dev/config-rev: "20260811-jetson-text-image-handoff" labels: app: hermes-model-gate spec: @@ -62,13 +62,19 @@ spec: ports: - name: http containerPort: 8080 + - name: handoff + containerPort: 8081 env: - name: UPSTREAM_URL - value: http://hermes-ollama.hermes.svc.cluster.local:11434 - - name: LEASE_NAMESPACE + value: http://ollama.ai.svc.cluster.local:11434 + - name: LOCAL_IMAGE_URL + value: http://hermes-local-image.hermes.svc.cluster.local:9004 + - name: HANDOFF_TIMEOUT_SEC + value: "1200" + - name: IMAGE_NAMESPACE value: hermes - - name: LEASE_NAME - value: titan-24-gpu-owner + - name: IMAGE_DEPLOYMENT + value: hermes-local-image readinessProbe: httpGet: path: /healthz @@ -125,3 +131,19 @@ spec: - name: http port: 11434 targetPort: http +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-gpu-handoff + namespace: hermes + labels: + app: hermes-model-gate +spec: + type: ClusterIP + selector: + app: hermes-model-gate + ports: + - name: http + port: 11434 + targetPort: handoff diff --git a/services/hermes/model-gate-rbac.yaml b/services/hermes/model-gate-rbac.yaml index 1c56556ac..592501dce 100644 --- a/services/hermes/model-gate-rbac.yaml +++ b/services/hermes/model-gate-rbac.yaml @@ -11,11 +11,11 @@ metadata: name: hermes-model-gate namespace: hermes rules: - - apiGroups: ["coordination.k8s.io"] + - apiGroups: ["apps"] resources: - - leases + - deployments resourceNames: - - titan-24-gpu-owner + - hermes-local-image verbs: - get --- diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index 786e08ad3..cf7456b7a 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -2,22 +2,43 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: hermes-ollama-ingress + name: hermes-local-image-ingress namespace: hermes spec: podSelector: matchLabels: - app: hermes-ollama + app: hermes-local-image policyTypes: - Ingress ingress: - from: - podSelector: - matchLabels: - app: hermes-model-gate + matchExpressions: + - key: app + operator: In + values: [hermes-agent, hermes-model-gate] ports: - - protocol: TCP - port: 11434 + - {protocol: TCP, port: 9004} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-model-gate-ingress + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-model-gate + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchExpressions: + - key: app + operator: In + values: [hermes, hermes-agent, hermes-chat-tenant] + ports: + - {protocol: TCP, port: 8080} - from: - namespaceSelector: matchLabels: @@ -26,8 +47,7 @@ spec: matchLabels: app: ariadne ports: - - protocol: TCP - port: 11434 + - {protocol: TCP, port: 8081} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/ollama-deployment.yaml b/services/hermes/ollama-deployment.yaml deleted file mode 100644 index 64a351f59..000000000 --- a/services/hermes/ollama-deployment.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# services/hermes/ollama-deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hermes-ollama - namespace: hermes - labels: - app: hermes-ollama -spec: - revisionHistoryLimit: 2 - strategy: - type: Recreate - selector: - matchLabels: - app: hermes-ollama - template: - metadata: - labels: - app: hermes-ollama - annotations: - ai.bstein.dev/model: gpt-oss:20b - ai.bstein.dev/gpu: titan-24 local-first lane - spec: - runtimeClassName: nvidia - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - titan-24 - volumes: - - name: models - persistentVolumeClaim: - claimName: hermes-models - initContainers: - - name: warm-model - image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d - imagePullPolicy: IfNotPresent - env: - - name: OLLAMA_HOST - value: 0.0.0.0 - - name: OLLAMA_MODELS - value: /root/.ollama - - name: OLLAMA_MODEL - value: gpt-oss:20b - - name: NVIDIA_VISIBLE_DEVICES - value: all - - name: NVIDIA_DRIVER_CAPABILITIES - value: compute,utility - command: - - /bin/sh - - -c - - | - set -e - ollama serve >/tmp/ollama.log 2>&1 & - sleep 6 - ollama pull "${OLLAMA_MODEL}" - pkill ollama || true - volumeMounts: - - name: models - mountPath: /root/.ollama - resources: - requests: - cpu: "1" - memory: 4Gi - nvidia.com/gpu.shared: 1 - limits: - cpu: "4" - memory: 16Gi - nvidia.com/gpu.shared: 1 - containers: - - name: ollama - image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 11434 - env: - - name: OLLAMA_HOST - value: 0.0.0.0 - - name: OLLAMA_KEEP_ALIVE - value: 6h - - name: OLLAMA_CONTEXT_LENGTH - value: "64000" - - name: OLLAMA_FLASH_ATTENTION - value: "1" - - name: OLLAMA_KV_CACHE_TYPE - value: q8_0 - - name: OLLAMA_MAX_LOADED_MODELS - value: "1" - - name: OLLAMA_NUM_PARALLEL - value: "1" - - name: OLLAMA_MODELS - value: /root/.ollama - - name: NVIDIA_VISIBLE_DEVICES - value: all - - name: NVIDIA_DRIVER_CAPABILITIES - value: compute,utility - volumeMounts: - - name: models - mountPath: /root/.ollama - readinessProbe: - httpGet: - path: /api/tags - port: 11434 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - resources: - requests: - cpu: "8" - memory: 24Gi - nvidia.com/gpu.shared: 1 - limits: - cpu: "16" - memory: 40Gi - nvidia.com/gpu.shared: 1 diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index 1cc1c0815..8fd5b4974 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -1,10 +1,11 @@ -"""Route Agent Hermes turns across Codex and Claude before inference begins.""" +"""Route Hermes turns across local, Codex, and Claude before inference begins.""" from __future__ import annotations import json import os import re +import threading import time import urllib.request from dataclasses import asdict, dataclass @@ -23,9 +24,12 @@ JETSON_MODEL = os.environ.get( "HERMES_AUTO_ROUTER_MODEL", "qwen2.5:3b-instruct-q4_0", ) +JETSON_WARM_URL = JETSON_URL.rsplit("/", 1)[0] + "/generate" EFFORTS = ("low", "medium", "high", "xhigh") -PROVIDERS = ("codex", "claude") +CHAT_MODE = os.environ.get("HERMES_AUTO_ROUTER_CHAT_MODE", "0") == "1" +PROVIDERS = ("codex", "claude", "local") if CHAT_MODE else ("codex", "claude") EFFORT_RANK = {effort: rank for rank, effort in enumerate(EFFORTS)} +_classifier_warm_lock = threading.Lock() try: PROVIDER_COOLDOWN_S = float( os.environ.get("HERMES_PROVIDER_COOLDOWN_S", "900") @@ -89,6 +93,30 @@ CONTEXTUAL_FOLLOWUP_PATTERNS = ( r"\bdo (?:it|that|this)\b", r"\bfinish (?:it|that|this|everything|all)\b", ) +IMAGE_ROUTE_TERMS = re.compile( + r"\b(?:draw|generate|image|illustration|photo|picture|portrait|render)\b", + re.IGNORECASE, +) +TEXT_PROVIDER_DIRECTIVES = { + "claude": ( + r"\b(?:ask|use|switch(?: me)? to|answer (?:using|with)|route (?:this )?to)\s+claude\b", + r"\bclaude\s+(?:should|must)\s+(?:answer|handle|do)\b", + ), + "codex": ( + r"\b(?:ask|use|switch(?: me)? to|answer (?:using|with)|route (?:this )?to)\s+(?:codex|openai)\b", + r"\b(?:codex|openai)\s+(?:should|must)\s+(?:answer|handle|do)\b", + ), + "local": ( + r"\b(?:answer|respond|run|do (?:this|it))\s+(?:entirely\s+)?locally\b", + r"\b(?:use|switch(?: me)? to|route (?:this )?to)\s+(?:the\s+)?(?:local|qwen)\s+(?:model|text|inference)\b", + r"\buse\s+qwen\b", + ), +} +TEXT_EFFORT_DIRECTIVE = re.compile( + r"\b(?:use|at|with|reasoning(?:\s+at)?|effort(?:\s+at)?)\s+" + r"(xhigh|extra[- ]high|high|medium|low)\b", + re.IGNORECASE, +) @dataclass(frozen=True) @@ -103,6 +131,54 @@ class Decision: latency_ms: int = 0 +def _explicit_text_override(text: str) -> tuple[str, str] | None: + """Parse a one-turn provider/effort directive without stealing image routes.""" + provider = "" + for candidate, patterns in TEXT_PROVIDER_DIRECTIVES.items(): + if any(re.search(pattern, text, re.IGNORECASE) for pattern in patterns): + provider = candidate + break + + # Image provider selection belongs to image_generate. In particular, a + # family-chat request for a "local image" must not also force the prose + # model to Qwen or reinterpret "OpenAI image" as a Codex text directive. + if provider in {"codex", "local"} and IMAGE_ROUTE_TERMS.search(text): + provider = "" + + effort = "" + effort_match = TEXT_EFFORT_DIRECTIVE.search(text) + if effort_match: + effort = effort_match.group(1).lower().replace("-", " ") + if effort == "extra high": + effort = "xhigh" + + if not provider and not effort: + return None + if provider == "local" and not CHAT_MODE: + provider = "" + return (provider, effort) if provider or effort else None + + +def _apply_explicit_text_override( + audit: Decision, override: tuple[str, str] | None +) -> Decision: + """Apply a one-turn instruction after retaining the Jetson audit result.""" + if override is None: + return audit + provider, effort = override + selected_provider = provider or audit.provider + selected_effort = effort or audit.effort + return Decision( + audit.shape, + selected_effort, + selected_provider, + f"explicit-{audit.classifier}", + "one-turn user override; Jetson audit suggested " + f"{audit.provider}/{audit.effort}", + audit.latency_ms, + ) + + def _tokens(text: str) -> set[str]: """Return lower-case words while retaining selected compound phrases.""" words = set(re.findall(r"[a-z0-9_-]+", text.lower())) @@ -295,7 +371,7 @@ def _jetson_scalar( "model": JETSON_MODEL, "stream": False, "format": {"type": "string", "enum": list(codes)}, - "keep_alive": "24h", + "keep_alive": "-1", "options": {"temperature": 0, "num_ctx": 512, "num_predict": 2}, "messages": [ {"role": "system", "content": prompt}, @@ -383,6 +459,14 @@ def classify_task( "heuristic-context", f"{baseline.reason}; resolved against recent assistant context", ) + if CHAT_MODE and baseline.shape == "question" and baseline.effort == "low": + return Decision( + baseline.shape, + baseline.effort, + "local", + baseline.classifier, + "bounded family-chat request suitable for local inference", + ) return baseline # The Jetson participates in every AUTO decision. Deterministic policy is a @@ -400,6 +484,8 @@ def classify_task( if baseline.shape in {"architecture", "review"} else local.provider ) + if CHAT_MODE and baseline.shape == "question" and effort == "low": + provider = "local" return Decision( shape, effort, @@ -480,6 +566,36 @@ def select_route( providers = status.get("providers") or {} policy = policy if isinstance(policy, dict) else _current_policy() selected = decision.provider + if CHAT_MODE: + routes = { + "local": ( + "custom/qwen2.5:14b-instruct-q4_0", + "atlas-codex/gpt-5.6-terra", + "anthropic/claude-sonnet-5", + ), + "codex": ( + "atlas-codex/gpt-5.6-terra", + "anthropic/claude-sonnet-5", + "custom/qwen2.5:14b-instruct-q4_0", + ), + "claude": ( + "anthropic/claude-sonnet-5", + "atlas-codex/gpt-5.6-terra", + "custom/qwen2.5:14b-instruct-q4_0", + ), + } + chain = routes.get(selected, routes["codex"]) + provider, model = _split_route(chain[0]) + if model_override: + model = model_override + return { + **asdict(decision), + "worker": selected, + "profile": f"chat-{selected}-{decision.effort}", + "provider": provider, + "model": model, + "fallback_chain": list(chain[1:]), + } provider_key = "openai-codex" if selected == "codex" else "anthropic" alternate = "claude" if selected == "codex" else "codex" alternate_key = "anthropic" if alternate == "claude" else "openai-codex" @@ -514,14 +630,7 @@ def _fallback_entry(route: str) -> dict[str, str]: """Expand a status route into Hermes' runtime fallback representation.""" provider, model = _split_route(route) entry = {"provider": provider, "model": model} - if provider == "custom" and model.startswith("qwen2.5"): - entry.update( - { - "base_url": "http://ollama.ai.svc.cluster.local:11434/v1", - "api_key": "ollama", - } - ) - elif provider == "custom": + if provider == "custom": entry.update( { "base_url": "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1", @@ -672,6 +781,44 @@ def _runtime_agent(ctx: Any) -> Any | None: return getattr(cli, "agent", None) if cli is not None else None +def _rewarm_classifier() -> None: + """Restore the small routing model after Qwen 14B used the sole GPU slot.""" + try: + payload = { + "model": JETSON_MODEL, + "prompt": "Reply with P", + "stream": False, + "keep_alive": "-1", + "options": {"temperature": 0, "num_ctx": 128, "num_predict": 1}, + } + request = urllib.request.Request( + JETSON_WARM_URL, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=180) as response: + json.load(response) + except Exception: + # The next AUTO turn still performs a real Jetson classification and + # retains the deterministic safety floor if the accelerator is down. + pass + finally: + _classifier_warm_lock.release() + + +def _rewarm_classifier_after_local(provider: str, model: str) -> None: + """Warm asynchronously so a local answer does not delay browser delivery.""" + if provider != "custom" or model != "qwen2.5:14b-instruct-q4_0": + return + if not _classifier_warm_lock.acquire(blocking=False): + return + threading.Thread( + target=_rewarm_classifier, + name="hermes-classifier-rewarm", + daemon=True, + ).start() + + def _post_turn_route(ctx: Any, **kwargs: Any) -> None: """Persist and surface the provider/model that completed the routed turn.""" policy = _current_policy() @@ -704,6 +851,7 @@ def _post_turn_route(ctx: Any, **kwargs: Any) -> None: ) policy["last_decision"] = last _write_policy(policy) + _rewarm_classifier_after_local(actual_provider, actual_model) emit = getattr(agent, "_emit_status", None) if not callable(emit): @@ -746,7 +894,10 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: ) plan = select_route(_load_json(ROUTING_PATH), decision, model) else: - decision = classify_task(text, kwargs.get("conversation_history")) + audit = classify_task(text, kwargs.get("conversation_history")) + decision = _apply_explicit_text_override( + audit, _explicit_text_override(text) + ) plan = select_route(_load_json(ROUTING_PATH), decision) _apply_route(ctx, agent, plan) _record_plan(policy, plan) @@ -757,6 +908,11 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: f"MANUAL target → {plan['provider']}/{plan['model']} · " f"{plan['effort']} · automatic capacity fallback remains enabled" ) + elif str(plan["classifier"]).startswith("explicit"): + emit( + f"USER target → {plan['provider']}/{plan['model']} · " + f"{plan['effort']} · one turn · automatic capacity fallback enabled" + ) else: source = { "jetson": "Jetson", @@ -909,7 +1065,7 @@ def _status_text(ctx: Any) -> str: f"Current runtime: {current}\n" f"Last requested route: {last_text}\n" f"Last actual outcome: {outcome_text}\n" - "Commands: /route auto | /route manual " + "Commands: /route auto | /route manual " " [model] | /route status" ) @@ -928,13 +1084,17 @@ def _route_command(ctx: Any, raw_args: str) -> str: return "AUTO routing enabled. The next task will be classified before inference.\n" + _status_text(ctx) if mode != "manual" or len(args) < 3: return ( - "Usage: /route auto | /route manual " + "Usage: /route auto | /route manual " " [model] | /route status" ) provider = args[1].lower() effort = args[2].lower() if provider not in PROVIDERS or effort not in EFFORTS: - return "Provider must be codex or claude; effort must be low, medium, high, or xhigh." + return ( + "Provider must be codex or claude" + + (" or local" if CHAT_MODE else "") + + "; effort must be low, medium, high, or xhigh." + ) model = args[3] if len(args) > 3 else "" decision = Decision("question", effort, provider, "manual", "explicit user override") plan = select_route(_load_json(ROUTING_PATH), decision, model) diff --git a/services/hermes/plugins/auto-router/plugin.yaml b/services/hermes/plugins/auto-router/plugin.yaml index 9c961998e..c3483ad54 100644 --- a/services/hermes/plugins/auto-router/plugin.yaml +++ b/services/hermes/plugins/auto-router/plugin.yaml @@ -1,6 +1,6 @@ name: auto-router version: "1" -description: Jetson-assisted provider, model, and reasoning-effort routing for Agent Hermes. +description: Jetson-assisted local, Codex, Claude, model, and reasoning-effort routing. provides_hooks: - pre_turn_route - pre_internal_route diff --git a/services/hermes/plugins/image-gen-broker/__init__.py b/services/hermes/plugins/image-gen-broker/__init__.py index d33e9e41f..50c992955 100644 --- a/services/hermes/plugins/image-gen-broker/__init__.py +++ b/services/hermes/plugins/image-gen-broker/__init__.py @@ -25,11 +25,21 @@ BROKER_URL = os.environ.get( "HERMES_IMAGE_BROKER_URL", "http://hermes-image-broker.hermes.svc.cluster.local:9002", ).rstrip("/") -DEFAULT_MODEL = "gpt-image-2-high" +DEFAULT_MODEL = "atlas-image-auto-high" MODELS = { + "atlas-image-auto-high": ( + "Auto (GPT Image quality, local fallback)", + "auto-high", + "Hosted quality first; FLUX fallback when hosted generation fails", + ), "gpt-image-2-low": ("GPT Image 2 (Fast)", "low"), "gpt-image-2-medium": ("GPT Image 2 (Balanced)", "medium"), "gpt-image-2-high": ("GPT Image 2 (Highest quality)", "high"), + "flux-2-klein-4b-local": ( + "Local FLUX 2 Klein 4B", + "local-high", + "Private local generation and editing on the shared RTX 3080", + ), } MAX_INPUT_BYTES = 25 << 20 @@ -125,16 +135,20 @@ class AtlasBrokerImageProvider(ImageGenProvider): return False def list_models(self) -> list[dict[str, Any]]: - return [ - { - "id": model, - "display": display, - "speed": "~30s–3min", - "strengths": "OpenAI GPT Image 2 quality", - "price": "included account capacity", - } - for model, (display, _quality) in MODELS.items() - ] + models = [] + for model, values in MODELS.items(): + display, _quality, *strengths = values + local = model == "flux-2-klein-4b-local" + models.append( + { + "id": model, + "display": display, + "speed": "~2–15min" if local else "~30s–3min", + "strengths": strengths[0] if strengths else "OpenAI GPT Image 2 quality", + "price": "local compute" if local else "included account capacity", + } + ) + return models def default_model(self) -> str: return DEFAULT_MODEL @@ -202,6 +216,9 @@ class AtlasBrokerImageProvider(ImageGenProvider): extra={ "quality": response.get("quality", MODELS[model][1]), "size": response.get("size"), + "route": response.get("route"), + "requested_model": response.get("requested_model", model), + "hosted_fallback_reason": response.get("hosted_fallback_reason"), "input_image_count": response.get( "input_image_count", len(references) + bool(primary) ), diff --git a/services/hermes/pvc.yaml b/services/hermes/pvc.yaml index 3a02cbfb7..1348719e2 100644 --- a/services/hermes/pvc.yaml +++ b/services/hermes/pvc.yaml @@ -62,10 +62,10 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: hermes-models + name: hermes-image-models namespace: hermes labels: - app: hermes-ollama + app: hermes-local-image spec: accessModes: - ReadWriteOnce diff --git a/services/hermes/scripts/hermes_model_routing.py b/services/hermes/scripts/hermes_model_routing.py index 47e3c2e46..69b47bd8d 100644 --- a/services/hermes/scripts/hermes_model_routing.py +++ b/services/hermes/scripts/hermes_model_routing.py @@ -20,7 +20,7 @@ CLAUDE_BASELINE = "claude-opus-5" EFFORTS = ("low", "medium", "high", "xhigh") ATLAS_FALLBACK = { "provider": "custom", - "model": "gpt-oss:20b", + "model": "qwen2.5:14b-instruct-q4_0", "base_url": "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1", "api_key": "ollama", } @@ -483,7 +483,7 @@ def configure_routes(root: Path, codex: Catalog, claude: Catalog) -> dict[str, A ), env_values, ) - local = [] if effort == "xhigh" else ["custom/gpt-oss:20b"] + local = [] if effort == "xhigh" else ["custom/qwen2.5:14b-instruct-q4_0"] routes[codex_name] = [ f"openai-codex/{codex_model}", f"anthropic/{claude_model}", @@ -523,6 +523,5 @@ def configure_routes(root: Path, codex: Catalog, claude: Catalog) -> dict[str, A routes["coordinator"] = [ f"openai-codex/{codex_coordinator}", f"anthropic/{claude_coordinator}", - "custom/gpt-oss:20b", ] return routes diff --git a/services/hermes/scripts/image_broker.py b/services/hermes/scripts/image_broker.py index 97b36c302..71ed7dba1 100644 --- a/services/hermes/scripts/image_broker.py +++ b/services/hermes/scripts/image_broker.py @@ -12,6 +12,8 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen HOST = os.environ.get("HERMES_IMAGE_BROKER_LISTEN_HOST", "0.0.0.0") @@ -26,20 +28,53 @@ PROVIDER_PATH = Path( ) ) DEFAULT_MODEL = os.environ.get( - "HERMES_IMAGE_BROKER_DEFAULT_MODEL", "gpt-image-2-high" + "HERMES_IMAGE_BROKER_DEFAULT_MODEL", "atlas-image-auto-high" +) +LOCAL_URL = os.environ.get( + "HERMES_LOCAL_IMAGE_URL", + "http://hermes-local-image.hermes.svc.cluster.local:9004", +).rstrip("/") +POLICY_PATH = Path( + os.environ.get("HERMES_IMAGE_POLICY_PATH", "/etc/hermes-image-policy/policy.json") ) MAX_BODY_BYTES = int(os.environ.get("HERMES_IMAGE_BROKER_MAX_BODY", str(96 << 20))) QUEUE_TIMEOUT_SECONDS = float( os.environ.get("HERMES_IMAGE_BROKER_QUEUE_TIMEOUT", "900") ) ALLOWED_MODELS = { + "atlas-image-auto-high", "gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high", + "flux-2-klein-4b-local", } ALLOWED_ASPECTS = {"landscape", "square", "portrait"} _generation_lock = threading.Lock() +# This is intentionally a tiny deterministic hard boundary, not a hosted-style +# appearance/race classifier. Operators can add site-specific phrases through +# the ConfigMap, but benign restoration and appearance correction remain valid. +_MINOR_TERMS = { + "baby", + "child", + "children", + "infant", + "kid", + "minor", + "preteen", + "toddler", + "underage", +} +_SEXUAL_TERMS = { + "erotic", + "explicit", + "genital", + "naked", + "nude", + "porn", + "sexual", +} + def _load_provider() -> Any: """Load the pinned Hermes provider without duplicating its API client.""" @@ -71,11 +106,94 @@ def _authorized(header: str | None) -> bool: return hmac.compare_digest(header[7:].strip(), TOKEN) +def _policy_error(prompt: str) -> str | None: + """Return the narrow hard-boundary reason, otherwise allow the request.""" + lowered = prompt.casefold() + words = {"".join(char for char in word if char.isalnum()) for word in lowered.split()} + if words & _MINOR_TERMS and words & _SEXUAL_TERMS: + return "sexual content involving minors is not permitted" + try: + configured = json.loads(POLICY_PATH.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + configured = {} + for phrase in configured.get("additional_blocked_phrases") or []: + if isinstance(phrase, str) and phrase.strip().casefold() in lowered: + return "request is blocked by the operator image policy" + return None + + +def _local_request(payload: dict[str, Any], timeout: float = 1800.0) -> dict[str, Any]: + """Call the fixed in-cluster FLUX endpoint without forwarding credentials.""" + request = Request( + f"{LOCAL_URL}/v1/images/generations", + data=json.dumps(payload).encode("utf-8"), + method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with urlopen(request, timeout=timeout) as response: # noqa: S310 - fixed internal URL + return json.loads(response.read()) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace")[:1000] + raise RuntimeError(f"local image service returned HTTP {exc.code}: {body}") from exc + except URLError as exc: + raise RuntimeError(f"local image service is unavailable: {exc.reason}") from exc + + +def _local_health() -> dict[str, Any]: + """Read local availability for the broker health and AUTO router.""" + request = Request(f"{LOCAL_URL}/health", method="GET") + try: + with urlopen(request, timeout=5) as response: # noqa: S310 - fixed internal URL + return json.loads(response.read()) + except Exception as exc: + return {"success": False, "available": False, "error": str(exc)} + + +def _generate_hosted( + payload: dict[str, Any], model: str, prompt: str, aspect: str +) -> dict[str, Any]: + """Generate through the owner's hosted GPT Image provider.""" + old_model = os.environ.get("OPENAI_IMAGE_MODEL") + try: + os.environ["OPENAI_IMAGE_MODEL"] = model + result = _provider().generate( + prompt, + aspect, + image_url=payload.get("image_url"), + reference_image_urls=payload.get("reference_image_urls"), + ) + finally: + if old_model is None: + os.environ.pop("OPENAI_IMAGE_MODEL", None) + else: + os.environ["OPENAI_IMAGE_MODEL"] = old_model + if not result.get("success"): + return result + image_path = Path(str(result["image"])) + try: + raw = image_path.read_bytes() + finally: + image_path.unlink(missing_ok=True) + response = {key: value for key, value in result.items() if key != "image"} + response.update( + { + "image_b64": base64.b64encode(raw).decode("ascii"), + "mime_type": "image/png", + "route": "openai", + } + ) + return response + + def _generate(payload: dict[str, Any]) -> dict[str, Any]: - """Generate once, return bytes, and remove the broker-side cache copy.""" + """Route one request and keep generated bytes inside the tenant boundary.""" prompt = str(payload.get("prompt") or "").strip() if not prompt: raise ValueError("prompt is required") + policy_error = _policy_error(prompt) + if policy_error: + raise ValueError(policy_error) aspect = str(payload.get("aspect_ratio") or "landscape").lower() if aspect not in ALLOWED_ASPECTS: raise ValueError("aspect_ratio must be landscape, square, or portrait") @@ -95,37 +213,37 @@ def _generate(payload: dict[str, Any]) -> dict[str, Any]: acquired = _generation_lock.acquire(timeout=QUEUE_TIMEOUT_SECONDS) if not acquired: raise TimeoutError("image generation queue is busy; retry shortly") - old_model = os.environ.get("OPENAI_IMAGE_MODEL") try: - os.environ["OPENAI_IMAGE_MODEL"] = model - result = _provider().generate( - prompt, - aspect, - image_url=image_url, - reference_image_urls=references, - ) - finally: - if old_model is None: - os.environ.pop("OPENAI_IMAGE_MODEL", None) - else: - os.environ["OPENAI_IMAGE_MODEL"] = old_model - _generation_lock.release() + if model == "flux-2-klein-4b-local": + return _local_request(payload) + if model != "atlas-image-auto-high": + return _generate_hosted(payload, model, prompt, aspect) - if not result.get("success"): + hosted_error = "hosted image generation was unavailable" + try: + result = _generate_hosted(payload, "gpt-image-2-high", prompt, aspect) + if result.get("success"): + result["requested_model"] = model + return result + hosted_error = str(result.get("error") or hosted_error) + except Exception as exc: + hosted_error = f"{type(exc).__name__}: {exc}" + try: + result = _local_request({**payload, "model": "flux-2-klein-4b-local"}) + except Exception as exc: + return { + "success": False, + "error": ( + f"OpenAI image route failed ({hosted_error}); local FLUX fallback " + f"also failed ({type(exc).__name__}: {exc})" + ), + "error_type": "all_image_routes_failed", + } + result["requested_model"] = model + result["hosted_fallback_reason"] = hosted_error return result - image_path = Path(str(result["image"])) - try: - raw = image_path.read_bytes() finally: - image_path.unlink(missing_ok=True) - response = {key: value for key, value in result.items() if key != "image"} - response.update( - { - "image_b64": base64.b64encode(raw).decode("ascii"), - "mime_type": "image/png", - } - ) - return response + _generation_lock.release() class Handler(BaseHTTPRequestHandler): @@ -154,12 +272,19 @@ class Handler(BaseHTTPRequestHandler): return if not self._check_auth(): return + local = _local_health() + try: + openai_available = bool(_provider().is_available()) + except Exception: + openai_available = False self._json( 200, { "success": True, - "provider": "openai-codex", - "available": bool(_provider().is_available()), + "provider": "atlas-image-router", + "available": openai_available or bool(local.get("available")), + "openai_available": openai_available, + "local": local, "busy": _generation_lock.locked(), }, ) diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index 8ea66ba36..c6dfe39dc 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -112,16 +112,16 @@ spec: apiVersion: v1 kind: Service metadata: - name: hermes-ollama + name: hermes-local-image namespace: hermes labels: - app: hermes-ollama + app: hermes-local-image spec: type: ClusterIP selector: - app: hermes-ollama + app: hermes-local-image ports: - name: http - port: 11434 - targetPort: http + port: 9004 + targetPort: local-image protocol: TCP diff --git a/services/maintenance/apps/ariadne-deployment.yaml b/services/maintenance/apps/ariadne-deployment.yaml index 8f56325de..b2cc8ca44 100644 --- a/services/maintenance/apps/ariadne-deployment.yaml +++ b/services/maintenance/apps/ariadne-deployment.yaml @@ -294,9 +294,9 @@ spec: - name: GAME_MODE_LEASE_NAME value: titan-24-gpu-owner - name: GAME_MODE_OLLAMA_URL - value: http://hermes-ollama.hermes.svc.cluster.local:11434 + value: http://hermes-gpu-handoff.hermes.svc.cluster.local:11434 - name: GAME_MODE_OLLAMA_MODEL - value: gpt-oss:20b + value: flux-2-klein-4b-local - name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC value: "900" - name: GAME_MODE_TRANSITION_TIMEOUT_SEC @@ -614,7 +614,7 @@ spec: - name: ARIADNE_TESTING_TRIAGE_MODEL_URL value: http://hermes-model-gate.hermes.svc.cluster.local:11434 - name: ARIADNE_TESTING_TRIAGE_MODEL - value: gpt-oss:20b + value: qwen2.5:14b-instruct-q4_0 - name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC value: "900" - name: JENKINS_WORKSPACE_NAMESPACE diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index 7eb0490f8..ade1efaa4 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -511,6 +511,56 @@ def test_manual_policy_is_reapplied_on_every_non_command_turn(monkeypatch): assert plans[0]["classifier"] == "manual-jetson" +def test_chat_natural_language_override_is_one_turn_and_keeps_jetson_audit( + monkeypatch, +): + monkeypatch.setattr(router, "CHAT_MODE", True) + monkeypatch.setattr(router, "PROVIDERS", ("codex", "claude", "local")) + monkeypatch.setattr(router, "_current_policy", lambda: {"mode": "auto"}) + monkeypatch.setattr(router, "_load_json", lambda path: {}) + calls = [] + monkeypatch.setattr( + router, + "classify_task", + lambda text, history=None: calls.append(text) + or router.Decision("question", "low", "local", "jetson", "audit", 8), + ) + plans = [] + monkeypatch.setattr(router, "_apply_route", lambda ctx, agent, plan: plans.append(plan)) + monkeypatch.setattr(router, "_record_plan", lambda policy, plan: None) + + class Agent: + def _emit_status(self, message): + self.message = message + + agent = Agent() + router._pre_turn_route( + object(), agent=agent, user_message="Use Claude at xhigh for this answer." + ) + + assert calls == ["Use Claude at xhigh for this answer."] + assert plans[0]["provider"] == "anthropic" + assert plans[0]["model"] == "claude-sonnet-5" + assert plans[0]["effort"] == "xhigh" + assert plans[0]["classifier"] == "explicit-jetson" + assert agent.message.startswith("USER target") + + +def test_chat_text_overrides_do_not_steal_image_provider_instructions(monkeypatch): + monkeypatch.setattr(router, "CHAT_MODE", True) + + assert router._explicit_text_override("Use local image generation for this photo") is None + assert router._explicit_text_override("Generate this image with OpenAI") is None + assert router._explicit_text_override("Answer locally with the Qwen model") == ( + "local", + "", + ) + assert router._explicit_text_override("Ask Codex with high reasoning") == ( + "codex", + "high", + ) + + def test_post_turn_records_and_announces_capacity_fallback(monkeypatch): policy = { "mode": "auto", @@ -547,6 +597,43 @@ def test_post_turn_records_and_announces_capacity_fallback(monkeypatch): assert agent.message.startswith("FALLBACK USED") +def test_post_turn_rewarms_classifier_after_local_chat(monkeypatch): + policy = { + "mode": "auto", + "last_decision": { + "provider": "custom", + "model": "qwen2.5:14b-instruct-q4_0", + "effort": "low", + "classifier": "jetson", + }, + } + warmed = [] + monkeypatch.setattr(router, "_current_policy", lambda: policy) + monkeypatch.setattr(router, "_write_policy", lambda value: None) + monkeypatch.setattr( + router, + "_rewarm_classifier_after_local", + lambda provider, model: warmed.append((provider, model)), + ) + + class Agent: + provider = "custom" + model = "qwen2.5:14b-instruct-q4_0" + + def _emit_status(self, message): + self.message = message + + agent = Agent() + cli = type("CLI", (), {"agent": agent})() + manager = type("Manager", (), {"_cli_ref": cli})() + ctx = type("Context", (), {"_manager": manager})() + + router._post_turn_route(ctx, model="qwen2.5:14b-instruct-q4_0") + + assert warmed == [("custom", "qwen2.5:14b-instruct-q4_0")] + assert agent.message.startswith("ROUTE USED") + + def test_status_distinguishes_requested_route_from_actual_outcome(monkeypatch): monkeypatch.setattr( router, diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 135b69b24..7abbd98dd 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -320,9 +320,9 @@ def test_chat_image_generation_uses_private_owner_broker(): config = yaml.safe_load(configmap["data"]["config.yaml"]) assert config["image_gen"] == { "provider": "atlas-broker", - "model": "gpt-image-2-high", + "model": "atlas-image-auto-high", } - assert config["plugins"]["enabled"] == ["atlas-broker"] + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] pod = statefulset["spec"]["template"]["spec"] @@ -428,7 +428,7 @@ def test_chat_reasoning_uses_private_owner_codex_broker(): statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] assert statefulset["spec"]["template"]["metadata"]["annotations"][ "ai.bstein.dev/config-rev" - ] == "20260811-codex-broker" + ] == "20260811-image-routing" hermes = next( item for item in statefulset["spec"]["template"]["spec"]["containers"] @@ -522,6 +522,149 @@ def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monk assert not generated.exists() +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) is None + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] + / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "qwen2.5:14b-instruct-q4_0" in config + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """The two Qwen weights must not overcommit Xavier unified memory.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + + def test_chat_auth_file_mount_survives_atomic_provider_refresh(): statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] containers = statefulset["spec"]["template"]["spec"]["containers"] diff --git a/testing/tests/test_hermes_model_gate.py b/testing/tests/test_hermes_model_gate.py index e6974b5f6..9a2daa637 100644 --- a/testing/tests/test_hermes_model_gate.py +++ b/testing/tests/test_hermes_model_gate.py @@ -37,3 +37,28 @@ def test_model_gate_preserves_supported_and_non_json_requests(): assert normalize(supported) == supported assert normalize(non_json) == non_json + + +def test_model_gate_runs_a_renderer_aware_ariadne_handoff(): + """Wolf cannot reach Ollama until the local image service reports idle.""" + namespace = _model_gate_namespace() + assert "_wait_for_image_idle" in namespace + assert namespace["HANDOFF_PORT"] == 8081 + handoff = namespace["HandoffHandler"] + assert callable(handoff.do_GET) + assert callable(handoff.do_POST) + + +def test_model_gate_can_only_read_the_local_image_deployment(): + """The dead-renderer escape hatch must not grant workload mutation access.""" + documents = list(yaml.safe_load_all((HERMES / "model-gate-rbac.yaml").read_text())) + role = next(document for document in documents if document.get("kind") == "Role") + + assert role["rules"] == [ + { + "apiGroups": ["apps"], + "resources": ["deployments"], + "resourceNames": ["hermes-local-image"], + "verbs": ["get"], + } + ]