From ef32843a76d72de63c93c0d183e25b4431522ca4 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 23 Aug 2026 15:59:43 -0300 Subject: [PATCH] fix(hermes-voice): finalize mobile audio and reduce latency --- dockerfiles/hermes-webui-atlas-voice.js | 23 ++++++++----------- services/hermes/voice-deployment.yaml | 2 +- .../probes/hermes_voice_instrument_probe.js | 5 +--- testing/tests/test_hermes_chat_quality.py | 7 +++--- testing/tests/test_hermes_handsfree_stt.py | 12 ++++++---- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index a5042f5e..f15a0ef4 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -223,8 +223,6 @@ const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm']; const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||''; const chunks=[]; - const preRoll=[]; - let initialChunk=null; let heardSpeech=false; let voiceFrames=0; let noiseFloor=0.008; @@ -235,13 +233,7 @@ recorder.ondataavailable=function(event){ if(!event.data||!event.data.size) return; if(event.data.type) recordedMime=event.data.type; - if(heardSpeech){chunks.push(event.data);return;} - // MediaRecorder's first timeslice owns the container initialization - // (EBML/Opus headers for WebM, and equivalent headers for Ogg/MP4). - // Keep it separately while bounding the actual audio pre-roll. - if(!initialChunk){initialChunk=event.data;return;} - preRoll.push(event.data); - while(preRoll.length>3) preRoll.shift(); + chunks.push(event.data); }; recorder.onstop=function(){ if(vadTimer){clearInterval(vadTimer);vadTimer=null;} @@ -254,7 +246,12 @@ if(!heardSpeech||!chunks.length){restartSoon(token,300);return;} transcribe(new Blob(chunks,{type:recordedMime||'audio/webm'}),token); }; - recorder.start(250); + // Ask the browser for one finalized container at stop. Android Chromium + // can emit timeslice fragments without a reusable EBML initialization + // header; concatenating those fragments made otherwise valid recordings + // intermittently unreadable by ffmpeg. A bounded 90-second Opus capture + // is small enough to retain as one browser-owned recording. + recorder.start(); const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1600',10)||1600); vadTimer=window.setInterval(function(){ if(!active||token!==generation||!recorder||recorder.state==='inactive') return; @@ -274,8 +271,6 @@ if(!heardSpeech&&voiceFrames>=3){ heardSpeech=true; lastSpeech=now; - if(initialChunk){chunks.push(initialChunk);initialChunk=null;} - while(preRoll.length) chunks.push(preRoll.shift()); }else if(heardSpeech&&voiceNow){ lastSpeech=now; } @@ -352,7 +347,9 @@ const text=cleanForSpeech(rows[rows.length-1].dataset.rawText||''); if(!text){restartSoon(token,250);return;} setState('speaking'); - const chunks=typeof window._splitForTTS==='function'?window._splitForTTS(text,280):[text]; + // Shorter first chunks lower time-to-first-audio; the next chunk is + // synthesized while the current one plays. + const chunks=typeof window._splitForTTS==='function'?window._splitForTTS(text,160):[text]; try{ let pending=fetchSpeech(chunks[0],language); for(let index=0;index3) preRoll.shift()" in voice_script + assert "recorder.start();" in voice_script + assert "recorder.start(250)" not in voice_script stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() assert "def _repetitive_token" in stt_server @@ -466,7 +467,7 @@ def test_voice_workloads_have_deliberate_xavier_placement(): # Keep the deployment neutral so the old and new immutable images can # cross one Flux rollout safely; each image owns its compatible default. assert "HERMES_TTS_VOICE" not in tts_env - assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2" + assert tts_env["HERMES_TTS_ONNX_THREADS"] == "4" assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" assert all("hostPath" not in volume for volume in stt["volumes"]) assert all("hostPath" not in volume for volume in tts["volumes"]) diff --git a/testing/tests/test_hermes_handsfree_stt.py b/testing/tests/test_hermes_handsfree_stt.py index f0a11f4b..7a2e6e6f 100644 --- a/testing/tests/test_hermes_handsfree_stt.py +++ b/testing/tests/test_hermes_handsfree_stt.py @@ -92,14 +92,15 @@ def test_fixture_reproduces_the_current_ebml_header_failure(tmp_path: Path): assert "EBML header" in result.stderr -def test_header_preserved_browser_webm_reaches_real_conversion_boundary( +def test_complete_browser_recording_reaches_real_conversion_boundary( tmp_path: Path, ): module = _patched_transcription_module(tmp_path) + _fixture, chunks = _media_chunks() prepared, error = _convert( module, tmp_path, - _late_speech_upload(preserve_header=True), + b"".join(chunks), ".webm", ) @@ -164,8 +165,11 @@ def test_local_conversion_accepts_browser_fallback_containers(tmp_path: Path): def test_browser_uses_actual_recorder_mime_and_supported_extensions(): source = VOICE_JS.read_text(encoding="utf-8") - assert "let initialChunk=null" in source - assert "if(initialChunk){chunks.push(initialChunk);initialChunk=null;}" in source + assert "recorder.start();" in source + assert "recorder.start(250)" not in source + assert "chunks.push(event.data)" in source + assert "initialChunk" not in source + assert "preRoll" not in source assert "let recordedMime=recorder.mimeType||mime||''" in source assert "if(event.data.type) recordedMime=event.data.type" in source assert "audio/mp4;codecs=mp4a.40.2" in source