From 17e2ecfcf605ce6dd3e460efe7d512bda24ae948 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 10 Aug 2026 01:47:51 -0300 Subject: [PATCH] fix(hermes): isolate workers and gate noisy speech --- dockerfiles/hermes-jetson-stt-server.py | 44 +++++++++- dockerfiles/hermes-webui-atlas-voice.js | 38 +++++++-- services/hermes/agent-deployment.yaml | 4 +- services/hermes/chat-statefulset.yaml | 4 +- services/hermes/deployment.yaml | 4 +- services/hermes/scripts/herdr_tab_router.py | 90 +++++++++++++++++++++ services/hermes/voice-deployment.yaml | 4 +- testing/tests/test_hermes_chat_quality.py | 47 +++++++++++ testing/tests/test_hermes_herdr.py | 85 +++++++++++++++++++ 9 files changed, 306 insertions(+), 14 deletions(-) diff --git a/dockerfiles/hermes-jetson-stt-server.py b/dockerfiles/hermes-jetson-stt-server.py index a10ca336e..1ecb9374f 100644 --- a/dockerfiles/hermes-jetson-stt-server.py +++ b/dockerfiles/hermes-jetson-stt-server.py @@ -6,6 +6,7 @@ from __future__ import annotations import cgi import json import os +import re import tempfile import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -23,6 +24,44 @@ MAX_AUDIO_BYTES = 30 * 1024 * 1024 MODEL_LOCK = threading.Lock() +def _repetitive_token(token: str) -> bool: + """Identify long periodic Whisper hallucinations caused by steady noise.""" + letters = "".join(re.findall(r"[a-z]+", token.lower())) + if len(letters) < 10: + return False + for period in range(1, 5): + pattern = letters[:period] + matches = sum( + character == pattern[index % period] + for index, character in enumerate(letters) + ) + if matches / len(letters) >= 0.86: + return True + return max(letters.count(character) for character in set(letters)) / len(letters) >= 0.78 + + +def _clean_transcript(result: dict) -> str: + """Drop noise-only segments and repetitive tokens while retaining speech.""" + segments = result.get("segments") + if not isinstance(segments, list): + segments = [{"text": result.get("text") or ""}] + kept: list[str] = [] + for segment in segments: + if not isinstance(segment, dict): + continue + text = str(segment.get("text") or "").strip() + if not text: + continue + no_speech = float(segment.get("no_speech_prob") or 0.0) + average_logprob = float(segment.get("avg_logprob") or 0.0) + if no_speech >= 0.55 and average_logprob <= -0.55: + continue + words = [word for word in text.split() if not _repetitive_token(word)] + if words: + kept.append(" ".join(words)) + return " ".join(kept).strip() + + def _json(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None: body = json.dumps(payload).encode("utf-8") handler.send_response(status) @@ -104,9 +143,12 @@ class SpeechHandler(BaseHTTPRequestHandler): fp16=torch.cuda.is_available(), condition_on_previous_text=False, temperature=0, + compression_ratio_threshold=2.0, + logprob_threshold=-0.8, + no_speech_threshold=0.5, verbose=False, ) - transcript = str(result.get("text") or "").strip() + transcript = _clean_transcript(result) _json(self, 200, {"text": transcript, "model": MODEL_NAME}) except Exception as exc: print(f"[stt] transcription failed: {exc}", flush=True) diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index 0519ba313..e83c35b74 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -108,24 +108,42 @@ setState('listening'); try{ const capture=await navigator.mediaDevices.getUserMedia({ - audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true}, + audio:(function(){ + const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true}; + const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{}; + if(supported.voiceIsolation) constraints.voiceIsolation=true; + return constraints; + })(), }); if(!active||token!==generation){capture.getTracks().forEach(function(track){track.stop();});return;} stream=capture; const Context=window.AudioContext||window.webkitAudioContext; audioContext=new Context(); const analyser=audioContext.createAnalyser(); - analyser.fftSize=1024; - audioContext.createMediaStreamSource(stream).connect(analyser); + analyser.fftSize=2048; + const highpass=audioContext.createBiquadFilter(); + highpass.type='highpass'; + highpass.frequency.value=140; + highpass.Q.value=0.7; + audioContext.createMediaStreamSource(stream).connect(highpass); + highpass.connect(analyser); const samples=new Uint8Array(analyser.fftSize); const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm']; const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||''; const chunks=[]; + const preRoll=[]; let heardSpeech=false; + let voiceFrames=0; + let noiseFloor=0.008; let lastSpeech=Date.now(); const started=Date.now(); recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined); - recorder.ondataavailable=function(event){if(event.data&&event.data.size) chunks.push(event.data);}; + recorder.ondataavailable=function(event){ + if(!event.data||!event.data.size) return; + if(heardSpeech){chunks.push(event.data);return;} + preRoll.push(event.data); + while(preRoll.length>3) preRoll.shift(); + }; recorder.onstop=function(){ if(vadTimer){clearInterval(vadTimer);vadTimer=null;} const recordedStream=stream; @@ -149,7 +167,17 @@ } const rms=Math.sqrt(energy/samples.length); const now=Date.now(); - if(rms>0.025){heardSpeech=true;lastSpeech=now;} + const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006); + const voiceNow=rms>speechThreshold; + if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);} + voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0); + if(!heardSpeech&&voiceFrames>=3){ + heardSpeech=true; + lastSpeech=now; + while(preRoll.length) chunks.push(preRoll.shift()); + }else if(heardSpeech&&voiceNow){ + lastSpeech=now; + } const finished=heardSpeech&&(now-lastSpeech)>=silenceMs; const timedOut=now-started>=90000; const idle=(!heardSpeech)&&(now-started)>=20000; diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 2ce2986cb..888049ae1 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260810-worker-spaces-permissions" + ai.bstein.dev/config-rev: "20260810-detached-worker-spaces" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -350,7 +350,7 @@ spec: requests: {cpu: 250m, memory: 512Mi} limits: {cpu: "2", memory: 4Gi} - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32 + image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index c0d52ff9f..bf4e6570a 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -28,7 +28,7 @@ spec: ai.bstein.dev/role: isolated-user-chat ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides - ai.bstein.dev/config-rev: "20260810-natural-jetson-voice" + ai.bstein.dev/config-rev: "20260810-noise-gated-voice" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-chat vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -226,7 +226,7 @@ spec: requests: {cpu: 250m, memory: 512Mi} limits: {cpu: "1", memory: 2Gi} - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32 + image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 3e8401e32..6b86fd8ee 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/model: anthropic/claude-opus-5, falling back to openai-codex/gpt-5.6-terra then local gpt-oss:20b ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: titan-21 preferred, Jetson preferred, arm64 fallback - ai.bstein.dev/config-rev: "20260808-dedicated-triage" + ai.bstein.dev/config-rev: "20260810-noise-gated-webui" # The Anthropic credential comes from Vault rather than a manually # created Secret. The role is declared in # services/vault/scripts/vault_k8s_auth_configure.sh and bound to @@ -345,7 +345,7 @@ spec: cpu: "2" memory: 4Gi - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32 + image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/services/hermes/scripts/herdr_tab_router.py b/services/hermes/scripts/herdr_tab_router.py index eaf78ecac..5ca3a2575 100644 --- a/services/hermes/scripts/herdr_tab_router.py +++ b/services/hermes/scripts/herdr_tab_router.py @@ -133,6 +133,94 @@ def route_unmanaged_tabs( return started +def _pane_number(pane_id: str) -> int: + """Return a stable numeric sort key for Herdr pane identifiers.""" + match = re.search(r":p(\d+)$", pane_id) + return int(match.group(1)) if match else 2**31 - 1 + + +def _worker_label( + tab_label: str, + pane: dict[str, Any], + agent_names: dict[str, str], +) -> str: + """Describe a detached worker using its parent tab, provider, and role.""" + pane_id = str(pane.get("pane_id") or "") + parent = _slug(tab_label) + raw_name = agent_names.get(pane_id, "").strip() + name = _slug(raw_name) if raw_name else "" + provider = _slug(str(pane.get("agent") or "worker")) + suffix = _slug(pane_id.split(":")[-1]) + if name and name not in {parent, f"{parent}-{suffix}"}: + label = name if name.startswith(f"{parent}-") else f"{parent}-{name}" + else: + label = f"{parent}-{provider}-{suffix}" + return label[:32].rstrip("-") + + +def isolate_worker_panes( + workspace_id: str, + run: Callable[[list[str]], dict[str, Any]] = _run_json, +) -> list[str]: + """Keep one coordinator pane per tab and detach every worker workspace.""" + tabs_payload = run([str(HERDR_BIN), "tab", "list"]) + panes_payload = run( + [str(HERDR_BIN), "pane", "list", "--workspace", workspace_id] + ) + agents_payload = run([str(HERDR_BIN), "agent", "list"]) + tab_labels = { + str(item.get("tab_id") or ""): str(item.get("label") or "session") + for item in _result_list(tabs_payload, "tabs") + if item.get("workspace_id") == workspace_id + } + agent_names = { + str(item.get("pane_id") or ""): str(item.get("name") or "") + for item in _result_list(agents_payload, "agents") + if item.get("workspace_id") == workspace_id + } + panes_by_tab: dict[str, list[dict[str, Any]]] = {} + for pane in _result_list(panes_payload, "panes"): + if pane.get("workspace_id") != workspace_id: + continue + tab_id = str(pane.get("tab_id") or "") + if tab_id: + panes_by_tab.setdefault(tab_id, []).append(pane) + + moved: list[str] = [] + for tab_id, panes in panes_by_tab.items(): + if len(panes) <= 1: + continue + # The oldest Hermes pane is the tab's stable coordinator. Directly + # spawned Codex/Claude workers and later Hermes helpers are detached. + primary = min( + panes, + key=lambda pane: ( + 0 if pane.get("agent") == "hermes" else 1, + _pane_number(str(pane.get("pane_id") or "")), + ), + ) + for pane in panes: + if pane is primary: + continue + pane_id = str(pane.get("pane_id") or "") + if not pane_id: + continue + run( + [ + str(HERDR_BIN), + "pane", + "move", + pane_id, + "--new-workspace", + "--label", + _worker_label(tab_labels.get(tab_id, "session"), pane, agent_names), + "--no-focus", + ] + ) + moved.append(pane_id) + return moved + + def run_loop(label: str, interval: float) -> None: """Continuously reconcile Agent tabs while allowing Herdr to own workers.""" while True: @@ -141,6 +229,8 @@ def run_loop(label: str, interval: float) -> None: if workspace_id: for pane_id in route_unmanaged_tabs(workspace_id): print(f"Started routed Hermes session in {pane_id}", flush=True) + for pane_id in isolate_worker_panes(workspace_id): + print(f"Detached worker pane {pane_id}", flush=True) except Exception as error: print(f"Agent tab routing retry: {type(error).__name__}: {error}", flush=True) time.sleep(interval) diff --git a/services/hermes/voice-deployment.yaml b/services/hermes/voice-deployment.yaml index e79452e8b..db35a68b6 100644 --- a/services/hermes/voice-deployment.yaml +++ b/services/hermes/voice-deployment.yaml @@ -31,7 +31,7 @@ spec: kubernetes.io/hostname: titan-21 containers: - name: stt - image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:341fd5c93d202d7368bbd74bd392f207c66e6d3968cc39f2d1684ec87a9d3899 + image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f imagePullPolicy: IfNotPresent ports: - {name: http, containerPort: 9000, protocol: TCP} @@ -42,7 +42,7 @@ spec: - {name: HERMES_STT_MODEL, value: small} - {name: HERMES_STT_CACHE, value: /opt/models/whisper} - {name: NVIDIA_VISIBLE_DEVICES, value: all} - - {name: NVIDIA_DRIVER_CAPABILITIES, value: compute,utility} + - {name: NVIDIA_DRIVER_CAPABILITIES, value: "compute,utility"} - {name: JETSON_JETPACK, value: "5"} startupProbe: httpGet: {path: /health, port: http} diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index ec2f84791..7f827b0ff 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import sys +from types import SimpleNamespace from pathlib import Path import yaml @@ -185,6 +187,47 @@ def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): assert "window._splitForTTS(text,280)" in voice_script assert "pending=fetchSpeech(chunks[index+1])" in voice_script assert "restartSoon(token,450)" in voice_script + assert "constraints.voiceIsolation=true" in voice_script + assert "highpass.frequency.value=140" in voice_script + assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script + assert "while(preRoll.length>3) preRoll.shift()" in voice_script + + stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() + assert "def _repetitive_token" in stt_server + assert "compression_ratio_threshold=2.0" in stt_server + assert "no_speech_threshold=0.5" in stt_server + + +def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch): + server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py" + spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace()) + monkeypatch.setitem( + sys.modules, + "torch", + SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)), + ) + monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace()) + spec.loader.exec_module(module) + + result = { + "segments": [ + { + "text": " ththththththththth Testing.", + "no_speech_prob": 0.12, + "avg_logprob": -0.2, + }, + { + "text": " background hum", + "no_speech_prob": 0.82, + "avg_logprob": -0.9, + }, + ] + } + + assert module._clean_transcript(result) == "Testing." def test_voice_models_are_baked_and_runtime_has_no_public_egress(): @@ -238,6 +281,10 @@ def test_voice_workloads_have_deliberate_xavier_placement(): assert stt["runtimeClassName"] == "nvidia" assert stt["securityContext"]["supplementalGroups"] == [44] stt_resources = stt["containers"][0]["resources"] + stt_env = { + item["name"]: item["value"] for item in stt["containers"][0]["env"] + } + assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility" assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1 assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1 assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"] diff --git a/testing/tests/test_hermes_herdr.py b/testing/tests/test_hermes_herdr.py index fa0baa659..cae53961e 100644 --- a/testing/tests/test_hermes_herdr.py +++ b/testing/tests/test_hermes_herdr.py @@ -295,6 +295,91 @@ def test_agent_tab_router_only_targets_coordinator_workspace(): assert calls == [[str(tab_router.HERDR_BIN), "workspace", "list"]] +def test_agent_tab_router_detaches_workers_without_closing_sessions(): + calls = [] + + def fake_run(command): + calls.append(command) + if command[1:3] == ["tab", "list"]: + return { + "result": { + "tabs": [ + { + "workspace_id": "w2", + "tab_id": "w2:t2", + "label": "cassandra", + } + ] + } + } + if command[1:3] == ["pane", "list"]: + return { + "result": { + "panes": [ + { + "workspace_id": "w2", + "tab_id": "w2:t2", + "pane_id": "w2:p2", + "agent": "hermes", + }, + { + "workspace_id": "w2", + "tab_id": "w2:t2", + "pane_id": "w2:p8", + "agent": "claude", + }, + { + "workspace_id": "w2", + "tab_id": "w2:t2", + "pane_id": "w2:p6", + "agent": "codex", + }, + { + "workspace_id": "w2", + "tab_id": "w2:t2", + "pane_id": "w2:p7", + "agent": "hermes", + }, + ] + } + } + if command[1:3] == ["agent", "list"]: + return { + "result": { + "agents": [ + {"workspace_id": "w2", "pane_id": "w2:p2", "name": "cassandra-p2"}, + {"workspace_id": "w2", "pane_id": "w2:p8", "name": "cassandra-claude-review"}, + {"workspace_id": "w2", "pane_id": "w2:p6", "name": "cassandra-codex-review"}, + {"workspace_id": "w2", "pane_id": "w2:p7", "name": "cassandra-p7"}, + ] + } + } + return {"result": {"ok": True}} + + moved = tab_router.isolate_worker_panes("w2", fake_run) + + assert moved == ["w2:p8", "w2:p6", "w2:p7"] + move_calls = calls[-3:] + assert move_calls[0][1:] == [ + "pane", + "move", + "w2:p8", + "--new-workspace", + "--label", + "cassandra-claude-review", + "--no-focus", + ] + assert move_calls[1][-2:] == ["cassandra-codex-review", "--no-focus"] + assert move_calls[2][-2:] == ["cassandra-hermes-p7", "--no-focus"] + assert not any("close" in command for command in move_calls) + + +def test_detached_worker_label_falls_back_to_parent_provider_and_pane(): + pane = {"pane_id": "w4:p9", "agent": "codex"} + + assert tab_router._worker_label("Work", pane, {}) == "work-codex-p9" + + def test_agent_ttyd_defers_identity_to_owner_only_oauth_boundary(): deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) containers = deployment["spec"]["template"]["spec"]["containers"]