fix(hermes): isolate workers and gate noisy speech
This commit is contained in:
jenkins
parent
4f986b6ccb
commit
17e2ecfcf6
@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
import cgi
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
@ -23,6 +24,44 @@ MAX_AUDIO_BYTES = 30 * 1024 * 1024
|
||||
MODEL_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _repetitive_token(token: str) -> bool:
|
||||
"""Identify long periodic Whisper hallucinations caused by steady noise."""
|
||||
letters = "".join(re.findall(r"[a-z]+", token.lower()))
|
||||
if len(letters) < 10:
|
||||
return False
|
||||
for period in range(1, 5):
|
||||
pattern = letters[:period]
|
||||
matches = sum(
|
||||
character == pattern[index % period]
|
||||
for index, character in enumerate(letters)
|
||||
)
|
||||
if matches / len(letters) >= 0.86:
|
||||
return True
|
||||
return max(letters.count(character) for character in set(letters)) / len(letters) >= 0.78
|
||||
|
||||
|
||||
def _clean_transcript(result: dict) -> str:
|
||||
"""Drop noise-only segments and repetitive tokens while retaining speech."""
|
||||
segments = result.get("segments")
|
||||
if not isinstance(segments, list):
|
||||
segments = [{"text": result.get("text") or ""}]
|
||||
kept: list[str] = []
|
||||
for segment in segments:
|
||||
if not isinstance(segment, dict):
|
||||
continue
|
||||
text = str(segment.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
no_speech = float(segment.get("no_speech_prob") or 0.0)
|
||||
average_logprob = float(segment.get("avg_logprob") or 0.0)
|
||||
if no_speech >= 0.55 and average_logprob <= -0.55:
|
||||
continue
|
||||
words = [word for word in text.split() if not _repetitive_token(word)]
|
||||
if words:
|
||||
kept.append(" ".join(words))
|
||||
return " ".join(kept).strip()
|
||||
|
||||
|
||||
def _json(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
handler.send_response(status)
|
||||
@ -104,9 +143,12 @@ class SpeechHandler(BaseHTTPRequestHandler):
|
||||
fp16=torch.cuda.is_available(),
|
||||
condition_on_previous_text=False,
|
||||
temperature=0,
|
||||
compression_ratio_threshold=2.0,
|
||||
logprob_threshold=-0.8,
|
||||
no_speech_threshold=0.5,
|
||||
verbose=False,
|
||||
)
|
||||
transcript = str(result.get("text") or "").strip()
|
||||
transcript = _clean_transcript(result)
|
||||
_json(self, 200, {"text": transcript, "model": MODEL_NAME})
|
||||
except Exception as exc:
|
||||
print(f"[stt] transcription failed: {exc}", flush=True)
|
||||
|
||||
@ -108,24 +108,42 @@
|
||||
setState('listening');
|
||||
try{
|
||||
const capture=await navigator.mediaDevices.getUserMedia({
|
||||
audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true},
|
||||
audio:(function(){
|
||||
const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
|
||||
const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
|
||||
if(supported.voiceIsolation) constraints.voiceIsolation=true;
|
||||
return constraints;
|
||||
})(),
|
||||
});
|
||||
if(!active||token!==generation){capture.getTracks().forEach(function(track){track.stop();});return;}
|
||||
stream=capture;
|
||||
const Context=window.AudioContext||window.webkitAudioContext;
|
||||
audioContext=new Context();
|
||||
const analyser=audioContext.createAnalyser();
|
||||
analyser.fftSize=1024;
|
||||
audioContext.createMediaStreamSource(stream).connect(analyser);
|
||||
analyser.fftSize=2048;
|
||||
const highpass=audioContext.createBiquadFilter();
|
||||
highpass.type='highpass';
|
||||
highpass.frequency.value=140;
|
||||
highpass.Q.value=0.7;
|
||||
audioContext.createMediaStreamSource(stream).connect(highpass);
|
||||
highpass.connect(analyser);
|
||||
const samples=new Uint8Array(analyser.fftSize);
|
||||
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm'];
|
||||
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
||||
const chunks=[];
|
||||
const preRoll=[];
|
||||
let heardSpeech=false;
|
||||
let voiceFrames=0;
|
||||
let noiseFloor=0.008;
|
||||
let lastSpeech=Date.now();
|
||||
const started=Date.now();
|
||||
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
||||
recorder.ondataavailable=function(event){if(event.data&&event.data.size) chunks.push(event.data);};
|
||||
recorder.ondataavailable=function(event){
|
||||
if(!event.data||!event.data.size) return;
|
||||
if(heardSpeech){chunks.push(event.data);return;}
|
||||
preRoll.push(event.data);
|
||||
while(preRoll.length>3) preRoll.shift();
|
||||
};
|
||||
recorder.onstop=function(){
|
||||
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
||||
const recordedStream=stream;
|
||||
@ -149,7 +167,17 @@
|
||||
}
|
||||
const rms=Math.sqrt(energy/samples.length);
|
||||
const now=Date.now();
|
||||
if(rms>0.025){heardSpeech=true;lastSpeech=now;}
|
||||
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
||||
const voiceNow=rms>speechThreshold;
|
||||
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
||||
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
||||
if(!heardSpeech&&voiceFrames>=3){
|
||||
heardSpeech=true;
|
||||
lastSpeech=now;
|
||||
while(preRoll.length) chunks.push(preRoll.shift());
|
||||
}else if(heardSpeech&&voiceNow){
|
||||
lastSpeech=now;
|
||||
}
|
||||
const finished=heardSpeech&&(now-lastSpeech)>=silenceMs;
|
||||
const timedOut=now-started>=90000;
|
||||
const idle=(!heardSpeech)&&(now-started)>=20000;
|
||||
|
||||
@ -24,7 +24,7 @@ spec:
|
||||
ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260810-worker-spaces-permissions"
|
||||
ai.bstein.dev/config-rev: "20260810-detached-worker-spaces"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
@ -350,7 +350,7 @@ spec:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: "2", memory: 4Gi}
|
||||
- name: webui
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
|
||||
@ -28,7 +28,7 @@ spec:
|
||||
ai.bstein.dev/role: isolated-user-chat
|
||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||
ai.bstein.dev/config-rev: "20260810-natural-jetson-voice"
|
||||
ai.bstein.dev/config-rev: "20260810-noise-gated-voice"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-chat
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
@ -226,7 +226,7 @@ spec:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: "1", memory: 2Gi}
|
||||
- name: webui
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
|
||||
@ -24,7 +24,7 @@ spec:
|
||||
ai.bstein.dev/model: anthropic/claude-opus-5, falling back to openai-codex/gpt-5.6-terra then local gpt-oss:20b
|
||||
ai.bstein.dev/role: testing-triage
|
||||
ai.bstein.dev/placement: titan-21 preferred, Jetson preferred, arm64 fallback
|
||||
ai.bstein.dev/config-rev: "20260808-dedicated-triage"
|
||||
ai.bstein.dev/config-rev: "20260810-noise-gated-webui"
|
||||
# The Anthropic credential comes from Vault rather than a manually
|
||||
# created Secret. The role is declared in
|
||||
# services/vault/scripts/vault_k8s_auth_configure.sh and bound to
|
||||
@ -345,7 +345,7 @@ spec:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
- name: webui
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:6eb16f76b236e10b210a31e69c7e68103cbf921892d813561d63f8aab0a98c32
|
||||
image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
|
||||
@ -133,6 +133,94 @@ def route_unmanaged_tabs(
|
||||
return started
|
||||
|
||||
|
||||
def _pane_number(pane_id: str) -> int:
|
||||
"""Return a stable numeric sort key for Herdr pane identifiers."""
|
||||
match = re.search(r":p(\d+)$", pane_id)
|
||||
return int(match.group(1)) if match else 2**31 - 1
|
||||
|
||||
|
||||
def _worker_label(
|
||||
tab_label: str,
|
||||
pane: dict[str, Any],
|
||||
agent_names: dict[str, str],
|
||||
) -> str:
|
||||
"""Describe a detached worker using its parent tab, provider, and role."""
|
||||
pane_id = str(pane.get("pane_id") or "")
|
||||
parent = _slug(tab_label)
|
||||
raw_name = agent_names.get(pane_id, "").strip()
|
||||
name = _slug(raw_name) if raw_name else ""
|
||||
provider = _slug(str(pane.get("agent") or "worker"))
|
||||
suffix = _slug(pane_id.split(":")[-1])
|
||||
if name and name not in {parent, f"{parent}-{suffix}"}:
|
||||
label = name if name.startswith(f"{parent}-") else f"{parent}-{name}"
|
||||
else:
|
||||
label = f"{parent}-{provider}-{suffix}"
|
||||
return label[:32].rstrip("-")
|
||||
|
||||
|
||||
def isolate_worker_panes(
|
||||
workspace_id: str,
|
||||
run: Callable[[list[str]], dict[str, Any]] = _run_json,
|
||||
) -> list[str]:
|
||||
"""Keep one coordinator pane per tab and detach every worker workspace."""
|
||||
tabs_payload = run([str(HERDR_BIN), "tab", "list"])
|
||||
panes_payload = run(
|
||||
[str(HERDR_BIN), "pane", "list", "--workspace", workspace_id]
|
||||
)
|
||||
agents_payload = run([str(HERDR_BIN), "agent", "list"])
|
||||
tab_labels = {
|
||||
str(item.get("tab_id") or ""): str(item.get("label") or "session")
|
||||
for item in _result_list(tabs_payload, "tabs")
|
||||
if item.get("workspace_id") == workspace_id
|
||||
}
|
||||
agent_names = {
|
||||
str(item.get("pane_id") or ""): str(item.get("name") or "")
|
||||
for item in _result_list(agents_payload, "agents")
|
||||
if item.get("workspace_id") == workspace_id
|
||||
}
|
||||
panes_by_tab: dict[str, list[dict[str, Any]]] = {}
|
||||
for pane in _result_list(panes_payload, "panes"):
|
||||
if pane.get("workspace_id") != workspace_id:
|
||||
continue
|
||||
tab_id = str(pane.get("tab_id") or "")
|
||||
if tab_id:
|
||||
panes_by_tab.setdefault(tab_id, []).append(pane)
|
||||
|
||||
moved: list[str] = []
|
||||
for tab_id, panes in panes_by_tab.items():
|
||||
if len(panes) <= 1:
|
||||
continue
|
||||
# The oldest Hermes pane is the tab's stable coordinator. Directly
|
||||
# spawned Codex/Claude workers and later Hermes helpers are detached.
|
||||
primary = min(
|
||||
panes,
|
||||
key=lambda pane: (
|
||||
0 if pane.get("agent") == "hermes" else 1,
|
||||
_pane_number(str(pane.get("pane_id") or "")),
|
||||
),
|
||||
)
|
||||
for pane in panes:
|
||||
if pane is primary:
|
||||
continue
|
||||
pane_id = str(pane.get("pane_id") or "")
|
||||
if not pane_id:
|
||||
continue
|
||||
run(
|
||||
[
|
||||
str(HERDR_BIN),
|
||||
"pane",
|
||||
"move",
|
||||
pane_id,
|
||||
"--new-workspace",
|
||||
"--label",
|
||||
_worker_label(tab_labels.get(tab_id, "session"), pane, agent_names),
|
||||
"--no-focus",
|
||||
]
|
||||
)
|
||||
moved.append(pane_id)
|
||||
return moved
|
||||
|
||||
|
||||
def run_loop(label: str, interval: float) -> None:
|
||||
"""Continuously reconcile Agent tabs while allowing Herdr to own workers."""
|
||||
while True:
|
||||
@ -141,6 +229,8 @@ def run_loop(label: str, interval: float) -> None:
|
||||
if workspace_id:
|
||||
for pane_id in route_unmanaged_tabs(workspace_id):
|
||||
print(f"Started routed Hermes session in {pane_id}", flush=True)
|
||||
for pane_id in isolate_worker_panes(workspace_id):
|
||||
print(f"Detached worker pane {pane_id}", flush=True)
|
||||
except Exception as error:
|
||||
print(f"Agent tab routing retry: {type(error).__name__}: {error}", flush=True)
|
||||
time.sleep(interval)
|
||||
|
||||
@ -31,7 +31,7 @@ spec:
|
||||
kubernetes.io/hostname: titan-21
|
||||
containers:
|
||||
- name: stt
|
||||
image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:341fd5c93d202d7368bbd74bd392f207c66e6d3968cc39f2d1684ec87a9d3899
|
||||
image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- {name: http, containerPort: 9000, protocol: TCP}
|
||||
@ -42,7 +42,7 @@ spec:
|
||||
- {name: HERMES_STT_MODEL, value: small}
|
||||
- {name: HERMES_STT_CACHE, value: /opt/models/whisper}
|
||||
- {name: NVIDIA_VISIBLE_DEVICES, value: all}
|
||||
- {name: NVIDIA_DRIVER_CAPABILITIES, value: compute,utility}
|
||||
- {name: NVIDIA_DRIVER_CAPABILITIES, value: "compute,utility"}
|
||||
- {name: JETSON_JETPACK, value: "5"}
|
||||
startupProbe:
|
||||
httpGet: {path: /health, port: http}
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
@ -185,6 +187,47 @@ def test_chat_voice_uses_private_jetson_services_and_shared_auto_route():
|
||||
assert "window._splitForTTS(text,280)" in voice_script
|
||||
assert "pending=fetchSpeech(chunks[index+1])" in voice_script
|
||||
assert "restartSoon(token,450)" in voice_script
|
||||
assert "constraints.voiceIsolation=true" in voice_script
|
||||
assert "highpass.frequency.value=140" in voice_script
|
||||
assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
|
||||
assert "while(preRoll.length>3) preRoll.shift()" in voice_script
|
||||
|
||||
stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
|
||||
assert "def _repetitive_token" in stt_server
|
||||
assert "compression_ratio_threshold=2.0" in stt_server
|
||||
assert "no_speech_threshold=0.5" in stt_server
|
||||
|
||||
|
||||
def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch):
|
||||
server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace())
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace())
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
result = {
|
||||
"segments": [
|
||||
{
|
||||
"text": " ththththththththth Testing.",
|
||||
"no_speech_prob": 0.12,
|
||||
"avg_logprob": -0.2,
|
||||
},
|
||||
{
|
||||
"text": " background hum",
|
||||
"no_speech_prob": 0.82,
|
||||
"avg_logprob": -0.9,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
assert module._clean_transcript(result) == "Testing."
|
||||
|
||||
|
||||
def test_voice_models_are_baked_and_runtime_has_no_public_egress():
|
||||
@ -238,6 +281,10 @@ def test_voice_workloads_have_deliberate_xavier_placement():
|
||||
assert stt["runtimeClassName"] == "nvidia"
|
||||
assert stt["securityContext"]["supplementalGroups"] == [44]
|
||||
stt_resources = stt["containers"][0]["resources"]
|
||||
stt_env = {
|
||||
item["name"]: item["value"] for item in stt["containers"][0]["env"]
|
||||
}
|
||||
assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility"
|
||||
assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1
|
||||
assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1
|
||||
assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"]
|
||||
|
||||
@ -295,6 +295,91 @@ def test_agent_tab_router_only_targets_coordinator_workspace():
|
||||
assert calls == [[str(tab_router.HERDR_BIN), "workspace", "list"]]
|
||||
|
||||
|
||||
def test_agent_tab_router_detaches_workers_without_closing_sessions():
|
||||
calls = []
|
||||
|
||||
def fake_run(command):
|
||||
calls.append(command)
|
||||
if command[1:3] == ["tab", "list"]:
|
||||
return {
|
||||
"result": {
|
||||
"tabs": [
|
||||
{
|
||||
"workspace_id": "w2",
|
||||
"tab_id": "w2:t2",
|
||||
"label": "cassandra",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
if command[1:3] == ["pane", "list"]:
|
||||
return {
|
||||
"result": {
|
||||
"panes": [
|
||||
{
|
||||
"workspace_id": "w2",
|
||||
"tab_id": "w2:t2",
|
||||
"pane_id": "w2:p2",
|
||||
"agent": "hermes",
|
||||
},
|
||||
{
|
||||
"workspace_id": "w2",
|
||||
"tab_id": "w2:t2",
|
||||
"pane_id": "w2:p8",
|
||||
"agent": "claude",
|
||||
},
|
||||
{
|
||||
"workspace_id": "w2",
|
||||
"tab_id": "w2:t2",
|
||||
"pane_id": "w2:p6",
|
||||
"agent": "codex",
|
||||
},
|
||||
{
|
||||
"workspace_id": "w2",
|
||||
"tab_id": "w2:t2",
|
||||
"pane_id": "w2:p7",
|
||||
"agent": "hermes",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
if command[1:3] == ["agent", "list"]:
|
||||
return {
|
||||
"result": {
|
||||
"agents": [
|
||||
{"workspace_id": "w2", "pane_id": "w2:p2", "name": "cassandra-p2"},
|
||||
{"workspace_id": "w2", "pane_id": "w2:p8", "name": "cassandra-claude-review"},
|
||||
{"workspace_id": "w2", "pane_id": "w2:p6", "name": "cassandra-codex-review"},
|
||||
{"workspace_id": "w2", "pane_id": "w2:p7", "name": "cassandra-p7"},
|
||||
]
|
||||
}
|
||||
}
|
||||
return {"result": {"ok": True}}
|
||||
|
||||
moved = tab_router.isolate_worker_panes("w2", fake_run)
|
||||
|
||||
assert moved == ["w2:p8", "w2:p6", "w2:p7"]
|
||||
move_calls = calls[-3:]
|
||||
assert move_calls[0][1:] == [
|
||||
"pane",
|
||||
"move",
|
||||
"w2:p8",
|
||||
"--new-workspace",
|
||||
"--label",
|
||||
"cassandra-claude-review",
|
||||
"--no-focus",
|
||||
]
|
||||
assert move_calls[1][-2:] == ["cassandra-codex-review", "--no-focus"]
|
||||
assert move_calls[2][-2:] == ["cassandra-hermes-p7", "--no-focus"]
|
||||
assert not any("close" in command for command in move_calls)
|
||||
|
||||
|
||||
def test_detached_worker_label_falls_back_to_parent_provider_and_pane():
|
||||
pane = {"pane_id": "w4:p9", "agent": "codex"}
|
||||
|
||||
assert tab_router._worker_label("Work", pane, {}) == "work-codex-p9"
|
||||
|
||||
|
||||
def test_agent_ttyd_defers_identity_to_owner_only_oauth_boundary():
|
||||
deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
|
||||
containers = deployment["spec"]["template"]["spec"]["containers"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user