From cfd8a75e95afecf7aded16a63e3cfb0270cad4f1 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 21 Aug 2026 11:12:15 +0000 Subject: [PATCH] fix(hermes-voice): preserve MediaRecorder container headers --- dockerfiles/Dockerfile.hermes-webui | 6 +- dockerfiles/hermes-webui-atlas-voice.js | 21 +- dockerfiles/hermes-webui-stt-patch.py | 37 ++++ .../fixtures/hermes-agent-9de9c25f/SOURCE.md | 12 ++ .../tools/transcription_tools.py | 44 ++++ testing/fixtures/mediarecorder/SOURCE.md | 18 ++ .../mediarecorder/chromium-webm-opus.json | 20 ++ .../mediarecorder/chromium-webm-opus.webm | Bin 0 -> 21749 bytes .../probes/generate_mediarecorder_fixture.js | 84 ++++++++ .../probes/hermes_voice_instrument_probe.js | 57 ++++- testing/tests/test_hermes_handsfree_stt.py | 203 ++++++++++++++++++ testing/tests/test_hermes_voice_instrument.py | 5 +- 12 files changed, 496 insertions(+), 11 deletions(-) create mode 100644 dockerfiles/hermes-webui-stt-patch.py create mode 100644 testing/fixtures/hermes-agent-9de9c25f/SOURCE.md create mode 100644 testing/fixtures/hermes-agent-9de9c25f/tools/transcription_tools.py create mode 100644 testing/fixtures/mediarecorder/SOURCE.md create mode 100644 testing/fixtures/mediarecorder/chromium-webm-opus.json create mode 100644 testing/fixtures/mediarecorder/chromium-webm-opus.webm create mode 100644 testing/probes/generate_mediarecorder_fixture.js create mode 100644 testing/tests/test_hermes_handsfree_stt.py diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui index 29e69ad3..1392f4b0 100644 --- a/dockerfiles/Dockerfile.hermes-webui +++ b/dockerfiles/Dockerfile.hermes-webui @@ -90,12 +90,14 @@ PY # Add the Atlas voice bridge as a narrow integration layer. It activates only # when a tenant's server-side STT capability reports the private Jetson route. COPY dockerfiles/hermes-webui-atlas-patch.py /tmp/hermes-webui-atlas-patch.py +COPY dockerfiles/hermes-webui-stt-patch.py /tmp/hermes-webui-stt-patch.py COPY dockerfiles/hermes-webui-telegram-project-patch.py /tmp/hermes-webui-telegram-project-patch.py COPY dockerfiles/hermes-webui-atlas-voice.js /opt/hermes-webui/static/atlas-voice.js COPY dockerfiles/hermes-webui-atlas-voice.css /opt/hermes-webui/static/atlas-voice.css COPY dockerfiles/hermes-webui-router-patch.py /tmp/hermes-webui-router-patch.py COPY dockerfiles/hermes-webui-router.js /opt/hermes-webui/static/atlas-router.js RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-atlas-patch.py +RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-stt-patch.py RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-telegram-project-patch.py RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-router-patch.py @@ -109,6 +111,7 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ && grep -Fq "'atlas/auto/maximum': 'Automatic · Maximum'" /opt/hermes-webui/static/panels.js \ && grep -Fq 'Atlas Jetson (private)' /opt/hermes-webui/static/index.html \ && grep -Fq 'HERMES_WEBUI_ATLAS_TTS_URL' /opt/hermes-webui/api/routes.py \ + && grep -Fq 'Audio conversion failed: upload is invalid' /opt/hermes/tools/transcription_tools.py \ && grep -Fq "capability.provider!=='local_command'" /opt/hermes-webui/static/atlas-voice.js \ && grep -Fq 'prefers-reduced-motion: reduce' /opt/hermes-webui/static/atlas-voice.css \ && grep -Fq 'id="voiceInstrumentStyles"' /opt/hermes-webui/static/index.html \ @@ -118,7 +121,8 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ && grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \ && /opt/hermes/.venv/bin/python -m py_compile \ /opt/hermes-webui/api/routes.py \ - /opt/hermes-webui/api/gateway_chat.py + /opt/hermes-webui/api/gateway_chat.py \ + /opt/hermes/tools/transcription_tools.py # Exercise the real server process in the target architecture before publish. RUN set -eu; \ diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index 29cbf0a1..c43aca0f 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -131,10 +131,17 @@ if(typeof window.send==='function') window.send(); } + function audioExtension(mimeType){ + const normalized=String(mimeType||'').toLowerCase(); + if(normalized.indexOf('ogg')>=0) return 'ogg'; + if(normalized.indexOf('mp4')>=0) return 'mp4'; + return 'webm'; + } + async function transcribe(blob, token){ if(!active||token!==generation) return; setState('transcribing'); - const ext=(blob.type||'').indexOf('ogg')>=0?'ogg':'webm'; + const ext=audioExtension(blob.type); const form=new FormData(); form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext})); try{ @@ -182,19 +189,26 @@ audioContext.createMediaStreamSource(stream).connect(highpass); highpass.connect(analyser); const samples=new Uint8Array(analyser.fftSize); - const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm']; + const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm']; const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||''; const chunks=[]; const preRoll=[]; + let initialChunk=null; let heardSpeech=false; let voiceFrames=0; let noiseFloor=0.008; let lastSpeech=Date.now(); const started=Date.now(); recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined); + let recordedMime=recorder.mimeType||mime||''; recorder.ondataavailable=function(event){ if(!event.data||!event.data.size) return; + if(event.data.type) recordedMime=event.data.type; if(heardSpeech){chunks.push(event.data);return;} + // MediaRecorder's first timeslice owns the container initialization + // (EBML/Opus headers for WebM, and equivalent headers for Ogg/MP4). + // Keep it separately while bounding the actual audio pre-roll. + if(!initialChunk){initialChunk=event.data;return;} preRoll.push(event.data); while(preRoll.length>3) preRoll.shift(); }; @@ -207,7 +221,7 @@ recorder=null; if(!active||token!==generation) return; if(!heardSpeech||!chunks.length){restartSoon(token,300);return;} - transcribe(new Blob(chunks,{type:mime||'audio/webm'}),token); + transcribe(new Blob(chunks,{type:recordedMime||'audio/webm'}),token); }; recorder.start(250); const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1600',10)||1600); @@ -229,6 +243,7 @@ if(!heardSpeech&&voiceFrames>=3){ heardSpeech=true; lastSpeech=now; + if(initialChunk){chunks.push(initialChunk);initialChunk=null;} while(preRoll.length) chunks.push(preRoll.shift()); }else if(heardSpeech&&voiceNow){ lastSpeech=now; diff --git a/dockerfiles/hermes-webui-stt-patch.py b/dockerfiles/hermes-webui-stt-patch.py new file mode 100644 index 00000000..a6029152 --- /dev/null +++ b/dockerfiles/hermes-webui-stt-patch.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Bound local-STT conversion errors in the pinned Hermes Agent source.""" + +import os +from pathlib import Path + + +ROOT = Path(os.environ.get("HERMES_AGENT_PATCH_ROOT", "/opt/hermes")) + + +def replace_exact(path: Path, before: str, after: str) -> None: + """Replace one exact upstream fragment, failing closed on image drift.""" + source = path.read_text(encoding="utf-8") + if source.count(before) != 1: + raise SystemExit(f"Hermes STT patch context changed in {path}: {before[:80]!r}") + path.write_text(source.replace(before, after, 1), encoding="utf-8") + + +transcription = ROOT / "tools/transcription_tools.py" +replace_exact( + transcription, + """ except subprocess.CalledProcessError as e: + details = e.stderr.strip() or e.stdout.strip() or str(e) + logger.error("ffmpeg conversion failed for %s: %s", file_path, details) + return None, f"Failed to convert audio for local STT: {details}" +""", + """ except subprocess.CalledProcessError as e: + details = e.stderr.strip() or e.stdout.strip() or str(e) + logger.error( + "ffmpeg conversion failed for %s: %s", file_path, details[-2000:] + ) + return None, ( + "Audio conversion failed: upload is invalid, incomplete, or uses an " + "unsupported codec" + ) +""", +) diff --git a/testing/fixtures/hermes-agent-9de9c25f/SOURCE.md b/testing/fixtures/hermes-agent-9de9c25f/SOURCE.md new file mode 100644 index 00000000..de2e910c --- /dev/null +++ b/testing/fixtures/hermes-agent-9de9c25f/SOURCE.md @@ -0,0 +1,12 @@ +# Hermes Agent STT fixture provenance + +This minimal module contains the exact `_prepare_local_audio` patch context +extracted from the agent image inherited by `Dockerfile.hermes-webui`: + +- Image: `registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107` +- OCI platform manifest: `sha256:9b4c00a25fd08f0df3bcb800d755bd48576fb65af5ab0df40058fbb78a147e43` +- Upstream Hermes revision: `9de9c25f620ff7f1ce0fd5457d596052d5159596` +- Full upstream `tools/transcription_tools.py` SHA-256: `d7d7df56b98dfadc0a7c08d5addd02db4e3106e6fd3b753e9241c23145eb2951` + +Tests execute the production patcher against this source fragment and then run +its real ffmpeg conversion boundary with browser-produced bytes. diff --git a/testing/fixtures/hermes-agent-9de9c25f/tools/transcription_tools.py b/testing/fixtures/hermes-agent-9de9c25f/tools/transcription_tools.py new file mode 100644 index 00000000..460b320a --- /dev/null +++ b/testing/fixtures/hermes-agent-9de9c25f/tools/transcription_tools.py @@ -0,0 +1,44 @@ +"""Exact pinned local-audio preparation fragment with its direct dependencies.""" + +import os +from pathlib import Path +import shutil +import subprocess +from typing import Optional + + +LOCAL_NATIVE_AUDIO_FORMATS = {".wav", ".aiff", ".aif"} +logger = __import__("logging").getLogger(__name__) + + +def _find_ffmpeg_binary() -> Optional[str]: + return shutil.which("ffmpeg") + + +def windows_hide_flags() -> int: + return 0 + + +def _prepare_local_audio(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]: + """Normalize audio for local CLI STT when needed.""" + audio_path = Path(file_path) + if audio_path.suffix.lower() in LOCAL_NATIVE_AUDIO_FORMATS: + return file_path, None + + ffmpeg = _find_ffmpeg_binary() + if not ffmpeg: + return None, "Local STT fallback requires ffmpeg for non-WAV inputs, but ffmpeg was not found" + + converted_path = os.path.join(work_dir, f"{audio_path.stem}.wav") + command = [ffmpeg, "-y", "-i", file_path, converted_path] + + try: + subprocess.run(command, check=True, capture_output=True, text=True, timeout=300, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags()) + return converted_path, None + except subprocess.TimeoutExpired: + logger.error("ffmpeg conversion timed out for %s", file_path) + return None, "Audio conversion for local STT timed out" + except subprocess.CalledProcessError as e: + details = e.stderr.strip() or e.stdout.strip() or str(e) + logger.error("ffmpeg conversion failed for %s: %s", file_path, details) + return None, f"Failed to convert audio for local STT: {details}" diff --git a/testing/fixtures/mediarecorder/SOURCE.md b/testing/fixtures/mediarecorder/SOURCE.md new file mode 100644 index 00000000..092c4fb8 --- /dev/null +++ b/testing/fixtures/mediarecorder/SOURCE.md @@ -0,0 +1,18 @@ +# Chromium MediaRecorder fixture provenance + +`chromium-webm-opus.webm` is the concatenation of ten successive 250 ms-ish +data blobs emitted by a real Chromium `MediaRecorder` for +`audio/webm;codecs=opus`; `chromium-webm-opus.json` records their exact byte +lengths so tests can replay each browser event. The recording has leading +silence followed by a 440 Hz signal, mirroring speech that begins after the +hands-free VAD pre-roll is full. + +- Browser: Chromium 140.0.7339.16 (Playwright build v1187, arm64) +- Generator: `testing/probes/generate_mediarecorder_fixture.js` +- Timeslice request: 250 ms +- Complete concatenated payload: WebM/Opus, accepted by ffmpeg 7.1.5 +- Complete payload SHA-256: `5585014ecad185511a727ccf47aa8b1ac70ea6f3efd7778309742bf88e6faf51` +- First five chunks are treated as pre-speech by the test model; the production + pre-roll retains only the most recent three. + +The fixture is generated data, not recorded speech. diff --git a/testing/fixtures/mediarecorder/chromium-webm-opus.json b/testing/fixtures/mediarecorder/chromium-webm-opus.json new file mode 100644 index 00000000..05fb724c --- /dev/null +++ b/testing/fixtures/mediarecorder/chromium-webm-opus.json @@ -0,0 +1,20 @@ +{ + "mime_type": "audio/webm;codecs=opus", + "timeslice_ms": 250, + "pre_speech_chunk_count": 5, + "chunk_sizes": [ + 204, + 70, + 70, + 70, + 70, + 3864, + 5796, + 4830, + 4830, + 1945 + ], + "payload_file": "chromium-webm-opus.webm", + "browser": "140.0.7339.16", + "generator": "testing/probes/generate_mediarecorder_fixture.js" +} diff --git a/testing/fixtures/mediarecorder/chromium-webm-opus.webm b/testing/fixtures/mediarecorder/chromium-webm-opus.webm new file mode 100644 index 0000000000000000000000000000000000000000..7d963589babbc35130c610c017eb6b3faab92a6b GIT binary patch literal 21749 zcmd6v1yt4B*7i4>?gr`Z6lp}dyE~K=q>+#==>|#ZE|Kn#M!LI08U#Ude;YmLT)pSq zbK^btdtcn@USl}KFP6cp7|jIv@CY+pJ^nmjdz00IEsjez^F)$nS62|jN?x)1P~u@5PU zNX^S%zuUGCAUn(l{3k(>pu77Y_baiVR=z*2dVgF&0DoF>{Iq)Y)2j5R)#o2qKnxPrv}X{GbiD&nVA-;XOOn4eZcKdroeS~dK%y7+Mg&GxsINQV!!4S4D4 zmm(>;$Z=RdPe7dvHw9U}AZ2%T;<36@Cy?1h3c-KNmv$rPuW{as82hy5mEo*1DtTBt z3f4{gzTuQhTH|AZ)4a$GL3A?L=ai*y|a+R>HC-Y zb(yGD!Mucc4v-Av?O+oz0MB~>ErSjUm_>QFSUM90h@Vx($5_#D1$r`@(YeKwoyN>RP=hX>)4Vh9lCdMi6F#4cCG?_EI@OuR-0Xl|C zWh&F+ff!$9u0~T6ox`M>XAAeF^9W+YA*E7k_e>XaUqI6j^3&;vHi;80Lt%BXKs$#+ zbXw3})SJ2Nz9>C!Zw_92Z-SJ6LKPgUXGmlYo7Qss)c~gXhKZB6@3M{nfJxc0Z0xRo z%=w-%r%ua)iIOSk7EgNDTW9q_iR!v_mA2p+V6J>bb>dT|ydrKq(Fy97sv!E}c}g6+mCZ;%{dYy9W>7XW03c z1btyUGQCRUDmSOxF5)fQjF>%024rM79C4!ZbKaF8&N;PEaElS3X5#xxo2TSDUcW^) zYYeX;p~Q(VVsEzH;s4mrh*RJy)uoMzXBNdxI52S2y^zNC0xx)i%yyN&!qMa~{mE`( zx;fMSLr5{WMAQ?lBH?WUjr_o@uJ$mGvB&~vldKiONonus3@mYt$r*G8IaQo6@+hgS z97ZRuonvxH1a!gN)GyWKT)hK)?D5C@UjF%B4=F5pt$w%))Z^}fk{Pv=<$Gp2Xsd6u&+ z?`CzFeXrsou!>7c>5FTZ7~(J|s!BMU8t&-L6*I|Tj;*&NraRv%_;!>4$2ktEF%%l- zm{W9cl-@2f;vhjkw#(QES)p&v&k%bNqnl%e6DrT&3BA_bK)rk+=dSlr;+z)<)y zhN=mV0o(-NjvxM|*>=lY1f(9GAvj_Lar1mATF;@UZL`ty@P<}1)nL|~Cq89!mJ&?No@4^pyBHW7#%J8?S1&(3S zfozg9IP*g01bmr@2He6(c4CJm+Ae;=$yq~A>O1GTuk?V1N(IrG^-#z1{_R-@uJ4iG zo@nRewrayqVZ5Nb>(}<5lZKLx5P3L_!TJ2X1fBHf9Cy~@0>99zn7RWIk%(Mzt|W)C z>>->_*>O!_p@~lRnsQr=jVw9k*!-dJ8}*+;Znpq{*Nr<(Cu7EX_qQRUqok)zXfdMG#~*jl;V6T|dm z_EcBB=SmO)%gWWjn7PzHb8jBl0EJ@AnPmm8)&13qEc~PAICsw`AENdbnC)#h z*k7Bx5DRM$N`@qnxsfu(Udk{w6M%aLYgSOUBLjEIdZqV<(#T>96W|;Y-U;P*LNfX} zsS*I{bWklKAr%=)CJy5`?Jwt%C%w~?56R`WZ|dOWbbhJO+0R;CFFVb{|MHyY7C_m1 zj{AO~vSTA2Vrbrw>xKmlpp=hiCkZ#it~Q9$MdP6NmApsv?(4~t0xY}+%~weY;1}t< zqsa*lXXah^g%8xE>ufJzIP-Sdnmb#*=qEs#=u=qB-byig=Eg6-D_tuI-}=}djO&qb z0N6fHFh&!PWg6&nCw|*{2YAt&V)eq?>kQIOcE|`%2DG#m`d!7J?p6HO&3R?)ppg#? zbFr$2c0X#9q{%wVntB{|8nG-6nD}(+wr>e&oBx6>D=N?K#WJ?5HJ}0?!ISJt)MEex zHi84)g@ce>I7##YKek8X*-?>cMs24QoNt2NRFRXz*Hz+6=Gj5YWf?0WOy9Ctq?lC( z^m^Q=7c*WZRMW-L$7sEeBcr{DomgoiV|#Wtv$uBD`djL5al|Lqgj>{aEhb6d(~QDJ zp5OBSG$LBN=ma-X{YQwJ~dv0s8d@dg(Uahg4 zRlMo7w#qoth$2hWr+=gLx>mGg*R9(wKwE>R1NNN)lSzpWr~b^7BWLPK%BO?7o>(*| zB@9iB@QN6!Ra>p5c!LxoYl;u3`bGF_p+*pgu0Kmry*OM=sh<>~p~anjU5(@2GDN7K zn{qccXix$5+76Oryh6(HgF?6({=_5{9=0^%=3Fb_POnv}J}ePnI@q5I5<@5G465#; z<-;+(U=z%{Mi@1bh^No~K$PThc5fz6!VZ`aoqsWt{+f9*J{_S-t7sztB;Js;INsr( zc}T*u;nC97J+)GQLGtJ~OnKe%BQzA{rbr9eef7yxtV|VAigY9#JYsdHwd=}BHn3bw zX!5#4H4n9(^IT+e-#ur#3FQ&5F|0G4gAIRFKvr+p^x0r_7(^-Kuxn^Hs$GV)$dIQ@ zmAYMf$S_Y?E#|Fhuo3ytZjrom6kk=pqsGN?*(o;`s}dmR{CNK1;AJo=)GK{!2yV!; z6%^M={xS_N|8i~rwQBg#ITB9ODT%_-B1;H7Je*vD7O}JmCs;rEh|gt^P9z=0_8a30 z@rHf?Ny%Op+Vxk4DyY{THi@5LQ|29FXQJpic6|#RT7A#pVcjxC-kNgT?JG;vsPU%92q(c}UMU6~mC2(jOPAnH)_5RzrbVQ2j@%HDU9Ko>_`}!edzv zdA=(~A8|@4i(@j`4V1v7_KP0#CG$KJI|Z}NC+Opa_)3)AHdV%I+VXD7>z77*mz_^sTk5ecM@W-nV=&y{+*e?AO2GhnT8@?__sX--cAB$RSP5c?hth^ zqK`-PGcF0i2BNlWFsoIhiVM=)}5}P&_gTUsX(K;sL>s@lOr-bZzn@$)#eUecBDKzvbuN653~o- zwr`oKXn`A?9~+P{A8;&;&q@2T{E3IS#&CY~kUqHIJmmgH4P*F=ivO{h=&e)3amD)B zjqWmyd1=8#?T?hfUcF0To$f=KR7+M~n0jdQT~czI=|B%gQ`WPQJa; zqbvtc7rIYEK;wtwi85Ulc8(sN9d_d5QbUb^yLT#iuMjsNh34#|vIp z^x1j?Gi!p+D6QO&H4!4XNetrnTvIL~T{K@YnNf`|L^VWja5J^+cT2KM_~=hw9@*;0 zY@A(J0VV}To;&Mw2B5=L?|pQxk2{2UMShJi7O%9{A4v{T|J_V}B9s1_x!M1=nY36a z#SOoW7{0)>0F}xVH}cS#UltGM_&;-)ih{O(hzN}n1oDD;CW(@c@zFD(lFNEZk#mHi z^rm661YSMj84ej#J=*iU$=xZBHgOgQW@(RT0qJ*2>Epxku4kOsYK5I-_39s zyLMf^a*JV}qtKzo3rrpebqu?E_73XQJaHj?FcvaKR;+4AI?9&CQ%<0AGhTG@xD6$# z0J7>EkN}pB7cv|ne+MMZBmWFY2y}h`lAdo@nO{BRuw!W+K-P|~4YP&QmIJ;X4E_V*$Kgf1U=PHY8 zNp4=rW+S*6-gRUXPKaJm3Y3cO;)U6=S#qWfmr1K%Zc3H>$U~bkMf8M|!+#G7I>nbg zLBHbQqnvrt%mjqYUAS^YwNu3|b26)Q^^>YsrgQqWJ{igv25$;APZwekJ_cWQ)+vSI z-Y6ZC9#OS@_J>8Aj3*zFc9k%!hW%Q3X}>Rcl&RFo+_Cg0K!WS`E0Fx=A>UM-davSt zOeSD65!Dq|s#f2-QobDtiWEcAEnj=s|0Rikzi%_`Q3YJc0k;`%MX)-@u5|Y{ol!F8 z@$?eX=}lp zudg>+W-fcV*K4W0CX^|68;WY5a8jxbly;7{Bh;6H0(ohWKlRQi$`;4oP*Qj+_|7xa zvqMbL0>TzP&~ZW{33Z#E7lFy--b|op4sl?^Gk-OcF0DT_6XLXcGU@+so5>f{Va4ux zFEyb692`!7Y61MX(*kpCX3odz%!3uIJn|Ar&D^%_CaAbbrE?VhJs!4I2&R`DYq^DQ z2s^Q;t&tw!NvVb1}~I4?No<@V1#oph_p(yG+z29$TbPnnx<*)Sb8rx`LIo2I6q0Ej3RR?{Fbr z3Avl&>iHw6GhyeiO-Oq%8M`A zp_c^cB4$}HQgn2^gJJxIg?56}M=d(CT(3xlD3>71_ETK9B3>`$1HZX&6o$%WhICik zbRVjBDq$9%-Otzb(apTdR8i7%|8)G1Qpga!ED5e5 zHy3p*-+;s{L13Ex-b0{i;i@n0J>(upe({ih6-d6Rc;;Tkf70FhUm+8pDfQoziH>Rc z)Ja$J43&pMfP|!CkECxppJ)k9|!c2i|Y)sNz?xFkz`L>W6 zT;?GL1E?Lvh>qXMgxCBRGI{LwACL(`&UZ2?|E;_Ao0-%;14x)1Y8=oy4h08LsS&^5 zwG*BM#O6jAJbNYtETqqr>EKt4QV~ae1K;{3BLB$*|5lh}D}Q*s@X!`@_jye!Zdgh2 zWRhQ0AAb~s(ic_v)d4Jz`KhJ)<7cl`HO0pXRD}@|vDJj1pD08zubAvLOhc1}y;*%i z7tI?|$wn7F#arN2F>7>MO$t6?U$2@F#9nK_@KiafVwsF2znYymO!usGV=UFXAbA7g zPDo|P7CK${n!TX8MA5-a9ek>v%E(tz4>lA0fYa}0qR^LHMD)^}`s=yM)BEmLdF)R> zl2ZK*NcKP*hrgN0U+f_p#w1`5ArdDMXsBVfl+UI}Mz)@Qq-GlRZl4ktYZyd|rYT&I z43{+-ZgWX^NVtS0vi$HYj@F#$F4JG#Eizm4fXBJSfh`Xqr&@<~k2ucNBx>wv?nh_*E^}sKV;4;Z*0HOQ$BRSy8+h z6R0-9|kGyV=p+=9Zv-7O_zW0Rk{TVr8AcDLe~qJHUa-FpZ+LhHfL-7WZU zTnvkOui}5!-TJr4F8X=*CKI*PFAM8SOy4}AB8F5GW2P*(_-$T-*fLLe2lXHyfcX9z&a}{9kWkU<_bG; zo1}_qn;nhkEe^8_CKKB-JlO-H>ed#v19RlsTZZ_8U&(~Mf6g)F&&b4I>-S{x-!YT_ z1V|n+GMIh?5@Cr|A3N_il3z1O!(1d0@C=^y240ygjfhEdWYPK30BE(ty7vMmizVdv z9n~_L2%MC3CP0_4S`v#|d9VcU-VG+#GUUHLEM8sfv^EQJ=g# zSWk=D!uk#*X5Y-@2Ov3L7XCj6B$|DTj$bSH?Gn;>;E=T+MC)!oRM9X!OdlIy*^Lt- zd=CXOd1(~2#aOThvWv}fFLYFyJ{@9db=MM(|59~>Tv4J5?=F{upHOh_sNQ?)FxP^6B7u6ja~p>Yw@q(X+e)22`@4k1 zo2%&$faFgT5=`>%3CWKha__~kiuWr1p}X~8CX@cs*B+Bd%9I+M0iRMD)s0H)dlP*z z0ltssZrY9P2FvN0>HX2m`a)?;7_1y)j$J?LTUpzb)}pl^Yf4XkK6z7{4vD5&h?vnS z8F--gwg(Tqd;5A$h6a|?@5I%2ct|`$9UP_P;kCzSthApN0=6RW!5XG1K zsKoX@D#>0D0-FhW+Gnz2#_2(?O(Ne3lC|N@Gwtk_00HVZ1urKvl9lWBpZR?2Zu#VB zdsJBd5|#LJ{hmzzVl(+qfrJ_kZ5+V(cR)hkv+U)R{63_2?c{>EGTnl`UJ}0#x0ksk zP7^b;QKHUlB~=qT2vDY@Zi?-Lg&BntXnR&)mWpna=mX4jMA?y&v zi5Y(fB)>!@OC z>>FkCmynXGXuNz*s?&UDrVur1;>8{jogd1;(^IL@W^DPc_bV)ar>vRF93Z3z z!U{M3L=Y`%^K-W5Ou)5Evb#=N9WWevQoQr6Y0IaC4$8ofuf*KVA6!@wRWjjI11Ek+ zNMwid|8YXnlkhDe0RxHTA9c52Lw`~6U*FyOACL)S!)wQ89i-1LnA7iFGdpf^76G1s zm3K$`1?5`FqHePO4fqOk@bO$DC?4qK1~IZNav#p37vyDUUl;ZBEO8vgu*0@}EtV>& z5#jdFCiWO#{v46hR2er3RnFIdxVa#v`;AOAX#sx;nf$K<36l=QZDi&i@fV&BQrzI- z+WpTqBTyeykX#+GBWxMez@7W#Qg7mNG6SLta(ekk9+zdR0`xC?8A{?RGh><^5m9lf zA4^l7#IHurGY;1Y^fC(Uf?9hv0H>ZB!V=LLPr6IrOnelUmVE_{ zUsyt&yU-?r3{Cm)dbaKjtsJCm{hQm4qX8}=( zcRHMxT)e3G+l$ym14O_9rg6!eVI3#bFR6WA$*jj(^Xrc`c%`Dd=U3x6RE3bfDET>p zRG1c+o=SvlvHzi%P!y8kM*OMOu6QEh_mncmJ78{+7GnlL;pM z|1UG?I2!oRfJEy$+o*Yx&SXa`vTIiNVNQ}I>{d$fr#4Hc8>O%y zW09srI&y3I0x^m`!?P7G{vfj@2rzD z<6fH8X&`p)P}&dQMK!VnkfVZ)=H$L+w@Q&bcS~&C0 z>O;7;t`+w8gk%#hs_|9Mnf{)A9v)-sa=_w%&ick>A#30aajS4cUCu8FN&I*_cU=~f zDZ}`L;(bB_29kON)n5`4aaV+djYn_T7Urk8vRSXAxuj0SDyY>6159l?-3E(v)NUx_H z>s!nsaPLWb_y@CFe;$dQ=6uOM$3FxTl+9sV+u>AKG!4=rnssNss1`b; z8J$_ayfQ{Rk`Zh*87;}!QDlqq%7Ly*^UaCC8!{go5jh1?ulZSwWZ*+~og(w-h=_qJ z4&v8OX|D?!x^$AEg6o0=Z_`^AcWL1U4^6#x zVzs}_Zkc_)Vp3Wb&JNNGZ2reU;{69v$zSXt{}GUgZJPc7BxPECFY6@wFZ(q(VO;R>VDM@;S)}jyHX}7%vgajj6!kp$eIN z^iCwpqn&JJ_DT$`SBVAd_+??78z$J|)uc}yR;_h>75 z*pNC`wRAi0^lWsR^WddHmDry#$!LwgLD==)sxwIH~bD*xRMOh4Wn#&`q%C!_N zh@g3qazL_c@ak*rsa@U|jNb-fS2;_&|sNdCiwL=;U93?wZ|Qp>B1T7$en zt9OdEaCu*4HosyFbrB$Y^3qvYDMpQ<^bfUee$~~kL_%Nht$gK7s|4K>OLv@Lp*h;N zu{np83GF!@G$cTm)4B&m+WRDbre{qAoRGlbgA!nC(?`zK5(apIkOa=!a zpVsJ=2ael!NYp;yp=~(0Fa?s#aYZSIHq?Pfw}|RG`CQ}2J6tr~MnPCZl?#TAdZUYU zMevia+$pmyi|W74Ze9EXGWlmw$$x(k`1e5a&u6!WJ}ixZXSc|oZF+v2-LiKw$Xy;& zyD53(6=tXtMFZpVQQ6GAxc`!qL#9&nY1M&<`+HxjI*K8dk916yH9*gruZ;hf{ZBM zdVnO zcC`@!=6jF5 zCmbKQg1bC@aoxC#o4whWFMWa+k-qxn(HfPnJh!%{?HukFF<+;#t@RDVY_Ri0&?^>%Diddm zziy;LN3is_EodN^m2WGJ)3LsUehlA@p`w2NK~uX`R<*`pWHLuQ(QVlR-;#$Kq&v7+ ztMM3zro26IxZ?i%|Ifc*17DMXmT6_$*kBWdP0f;ujk!L7X2m68R_oJ&93f87=4_JT`@BHcAqc(V@+u!JnR+&Q7&avz zq4h#3LY7U{ng{u%4?8=bbxl(fgfN-v@4zi|VXQvLAcB7Ok_n@(S8;m(OqL!4`oxkj z>QjY>GFv>a;0I}I;fMUaDS_m@MlTo47d%W&yKOnhGgn98QNTHmKkofxpn~t;$ZmS5@-b7MTB{Rir9I}!dz|=Eu$HMI?bA< zyCg}$2x;K(#1nw;4QY^`F69lF0AQ@eo;IY>0DvF>)^K9UBiee4Wupbvws3R~ z3NELqUbza`-r4X{GW@Z3wpg#dmJvQ>nu_7AkkKgVvxQB-`8pRT@l7`COcagchUB43 ztJ5c#v9?)3-(=SqMPWy^uHJNK-UOjBWOAqr3g|o?sbkC?fDEnKo+xEo|4eX9Sw=%i zzP5tAYYZ%S!mj(w9x`KI(3FIE%a+H{r6CG-foQkPS&UZctG!D$S9@66UO46!`8kBB zk8W*7NeN@;D1-7#f*XS?KmX*&!*n;wy5{jV>w+-MmN72qG3E|Ndw=*Dp$ragZTd%g zfw=26MqS`XlWD|f4WtnQ=0#|VeFjZt`=jofeSG)~|FD9s3cNcxI>sYR)hGa(`m`+o z6j;KKwpRca0Ay9);_Ei>8+bdZ%S8oDZwN1M#8TDa^rO;Un(WD-E9V0a>F>ggJX_d^%4$c13ZPuCSLzZ%mpVAi>3w|T0a+&l z+@ZTX7`8*h`@oEYl4C*EO6XUmt5g(^fHn}p^3YG8 zg{yK>>P-O%gOJ%RV{mvKs^PpBxef`=6rM`vMJO>Io>4i)INqyDBN~X4LS)>hn~Lqs zXg*OWoTNzXm7mPzh0yKhy)G=vRI_5y`6 zR0O#!tZogq0>gxSpS?@B88?|LD;_r(uDgXvr0eR*x8o6oM5zM$UjxC2E%!v|rM=hHpOYHm6^C!k{j$ zhJ-c!mCCtm;G!VriEAcLK=XUQ$lJMt47i7ESknklIuPE5!0$pO2JmmcT8V7BDt0*S znU4Q2#TgtGiXd7Bha1OOIrFH*$4g7uC7f$ksx zk4XqXRyj1!A$LU^(@x+u967(KxDc#ja2>la51+s3(6!UcU6r{T>u1j;$q_up z3(2fyj`mDLJl)U(uhWc_Co!`VX_a%7nm+KXO;_$osK^s9sp1Fl#%imEla zg6(N!bcC>+zEElb-F?If+MFTKS3NFtC`o+N&3WnXhRsyQY;v6nK^E&JkVn}4F5z34 z%k0=&;gI7YNRK-JLa224D@HFzH3B|7I1P82NvvFdxj?LuGzk6nWV09p~? zb^(?#8frV}d<@c3aL(_xAhr7uYelXtd!%hza9h+G*dF$KU%~@95o}zLhL?Pr= z$(X+dqV?yN^jdYZ*47wdFzP(g)@4n-1LnRRe_hP>0Bk{U{c?wNt)tt31%SuW z24Udx;nUH%(>$l7xrD#6BCSF;7h68RFJGhqp5fLKhN#+@aF4}BDSq(Ss1vV2VF&4Nt0zw-fH09JIY-!~j*@hv~;8vudeH~r?Qp@H=at%#tz*}X#uz7fL4V8$f0 zKLV|w!SA?Np`KSWl^PsGdkRxqZv=l+@z%YHFAzqyP4eZ{pwtLCHMRhNzN_lIF$?NW z1|Ho9w17UhH)Kod%T3N5n(CO)m;>w99f#*0jpTM)*t`&RQ%F}24hYf$o8)FM5g_zjL@;bL76J`LLZa^SOvmXmZ>&- z<)`37?+5z;%)L81^VR%TyzfQ=p$guQSoI&n$GE{8l34QMbE06fz*i{9D+5q8xmQ*C zZ8_)h`33kxQ|`<6DfMt!ClpI~6LPKf4j%uwJ*k1mB2a~b5t5%s+5^e}y*Pi&So*3j zp`-sXXH^FlB8n!2>N`byEp@kp<5Y~4ok%nx5jQw#m5T*Y-PJPni&0dzC`A3fLF~6G zCY8uFB()Eav(dsUT6sA7^U*v!Y3v|Li7uWmQof|7(_wh_1oewa>HIBU2b5-O1QDW2 zD%sqo?R%zBR>Ph-TZ8(>y#hgUNYx9K;?jg_Ie_v6T;~}qBLd-b3a+;(XesKWNz1>s?%%AOy)V*o@OL=4J_i;vc5Q5Kwzd4dJ~m1w{_A zDPN3=e8h6o4>@^9Rd+dK9?)RmaW(*zsoR0zV0jY1UJ+v9q36B{D*23`N|=tiCnX4qlQBi6zhW33i}gf;g{2^ z2q}&wxTkKzH1tg$TgVQ2l~@tiO3LHOS2_wqFZmr8cE@ESV^+($Aoxj4kB}~~(48>QST`k)dS?ay{>y72Na}Qf=DP;`a5Pa5Q3p!mtOW*+m1LuB%b-K%l zX40%S3hPAtSu3e?CVMI(?XX(tNx`@=x{Ya<=JmWqiEOwe;SMJBjW;^aF&sqDn^R_p zuJEu>>d(`Bbu2Nj{O#P7Jj42}KR&tm=vnuM2jxw&eOF%^fX4(a z)Q^qUJG1`PF7B#dRBjc}4t0(m&X+9|!ia5ae|9mko#0}}Dsde($tcJtErRKIU zoFEYiYUS~2+_QyAgY``{kZ3N~sO{G79~7?w>jiidcEdzWvW-)%3iAAVUjf_4eHrgt z2h3-)^9SEc_8W!Jd*`U(66Dx|&qY+0FJ!E6D^L)j?d}j+4+oT34ywzUUcAkD@%ZsW zUt_Q9nRSHm@ztSvPj7J>Ou=o@^tDe^sm9B+0BrcGKGPKurBsumUWLU6m?1bYI{eQb z^A>euC<4rmH=~~0*=s)*%^I`MVh?VU?N=2&fMH z17J3D+lkx>18bN-()e`z7{`GO!wA&~Qo)$D#fw4Ep6f`HuS>yZ0pBnT{#YXok2G>ztE818+`Q6wuPcPw=4o9ax)n(jPH4@ZO ze-J|VEKv&1iWZeGDWrls1fRvZ%cGth#Jo1b56VRngy*l>@w5{ZSPQ-0KXw%$nd%UG zrDIbSSy-ai1~;q@% zouZK|qhV>}A-#8A;pE}>IwI!N*}RSSgGG|H-QT|7hID3J zo_gRqD!*Uxtjho{ZMh|aFDe(~B;yaj_2m3^7BV4l{#hOGFMjw!j0fj`Ct_?tW%|&2 zBUS^zCR|m1`e7R?BI>+APJz6{fSlf&YMuAGiJRTnrg(HsVln}XT7bqN6#=OQlao8f z48UVcV#TVjzyHI4BGXcoW6SZ<2lUI4c3<_|PNGdJoF@`@- z=4tB1hr1$6`a<*iy-k)k+=4oe3|gGudI~=@I$>#Gyh-aB*3`#lpQ(3#5h*=vp)JQ$ z9w$`^3qzJyuu4Y#q=Bpl($}mskT{{P2wng>MB+gi=BFZi&cl8&h6;4nX~;%icFD zyMa>kRU%{+0PtBnpP*w_b!$zRCP!Yovcq*8OD}>4MvN6%#`SBGN_@WMVWFO+o-%ze zL0+CKeT1m^(VAzS@-v|QWA!KEF@3s^1ZSw$OgfZEzZPIW6}E|#!&v?+kwtn&$0h35Ev2I}^}Xn?E7#{h$JU%iKuVKx~dT7%2^D`blW zpW|Woe4Yv&n$>&5dZTJwy}V1j-+fuj#%!ZD(YB8q{m^Q#x&t_#eh zS3cM`z-ECh9wo!)QU0xZtdpY3^g-WFMqsq+uPl;15h)k}h)7H5H)zABAnXEJk#a^M Zm(tu8(7C55gFiPEAsH;<@1G+2{{S9=Qn~;D literal 0 HcmV?d00001 diff --git a/testing/probes/generate_mediarecorder_fixture.js b/testing/probes/generate_mediarecorder_fixture.js new file mode 100644 index 00000000..746e7778 --- /dev/null +++ b/testing/probes/generate_mediarecorder_fixture.js @@ -0,0 +1,84 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { chromium } = require('playwright'); + +async function main() { + const outputPath = process.argv[2]; + if (!outputPath) throw new Error('usage: node generate_mediarecorder_fixture.js OUTPUT.json'); + + const browser = await chromium.launch({ + headless: true, + args: ['--autoplay-policy=no-user-gesture-required'], + }); + try { + const page = await browser.newPage(); + const fixture = await page.evaluate(async () => { + const mimeType = 'audio/webm;codecs=opus'; + if (!MediaRecorder.isTypeSupported(mimeType)) { + throw new Error(`${mimeType} is not supported by this Chromium build`); + } + + const context = new AudioContext({sampleRate: 48000}); + const destination = context.createMediaStreamDestination(); + const oscillator = context.createOscillator(); + const gain = context.createGain(); + oscillator.frequency.value = 440; + gain.gain.value = 0; + oscillator.connect(gain).connect(destination); + oscillator.start(); + + const chunks = []; + const recorder = new MediaRecorder(destination.stream, {mimeType}); + recorder.ondataavailable = event => { + if (event.data && event.data.size) chunks.push(event.data); + }; + const stopped = new Promise(resolve => { recorder.onstop = resolve; }); + const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); + + recorder.start(250); + await delay(1400); + gain.gain.setValueAtTime(0.35, context.currentTime); + await delay(900); + gain.gain.setValueAtTime(0, context.currentTime); + await delay(500); + recorder.stop(); + await stopped; + + oscillator.stop(); + await context.close(); + const encoded = []; + for (const chunk of chunks) { + const bytes = new Uint8Array(await chunk.arrayBuffer()); + let binary = ''; + for (let index = 0; index < bytes.length; index += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + } + encoded.push(btoa(binary)); + } + return { + mime_type: recorder.mimeType, + timeslice_ms: 250, + pre_speech_chunk_count: 5, + chunks_base64: encoded, + }; + }); + const chunks = fixture.chunks_base64.map(value => Buffer.from(value, 'base64')); + const payloadPath = outputPath.replace(/\.json$/i, '.webm'); + fixture.chunk_sizes = chunks.map(chunk => chunk.length); + fixture.payload_file = path.basename(payloadPath); + delete fixture.chunks_base64; + fixture.browser = await browser.version(); + fixture.generator = 'testing/probes/generate_mediarecorder_fixture.js'; + fs.writeFileSync(payloadPath, Buffer.concat(chunks)); + fs.writeFileSync(outputPath, `${JSON.stringify(fixture, null, 2)}\n`); + } finally { + await browser.close(); + } +} + +main().catch(error => { + console.error(error.stack || error); + process.exitCode = 1; +}); diff --git a/testing/probes/hermes_voice_instrument_probe.js b/testing/probes/hermes_voice_instrument_probe.js index 06b705ff..c5f7900e 100644 --- a/testing/probes/hermes_voice_instrument_probe.js +++ b/testing/probes/hermes_voice_instrument_probe.js @@ -2,6 +2,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); +const path = require('node:path'); const vm = require('node:vm'); class StyleDeclaration { @@ -92,6 +93,7 @@ async function boot(scriptPath, reduced) { const reducedMotion = {matches: reduced, addEventListener() {}, removeEventListener() {}}; let rejectNextCapture = false; let transcriptResolve; + let uploadedFile = null; let sent = 0; let lastAudio = null; @@ -117,9 +119,10 @@ async function boot(scriptPath, reduced) { } class FakeMediaRecorder { static isTypeSupported() { return true; } - constructor(stream) { + constructor(stream, options) { this.stream = stream; this.state = 'inactive'; + this.mimeType = options?.mimeType || 'audio/webm;codecs=opus'; this.ondataavailable = null; this.onstop = null; recorders.push(this); @@ -171,11 +174,18 @@ async function boot(scriptPath, reduced) { }, }, }; - async function fetch(url) { + async function fetch(url, options) { if (url === '/api/transcribe/capability') { return {ok: true, json: async () => ({available: true, provider: 'local_command'})}; } if (url === '/api/transcribe') { + const file = options?.body?.get('file'); + assert.ok(file, 'transcription request did not carry a file'); + uploadedFile = { + bytes: new Uint8Array(await file.arrayBuffer()), + name: file.name, + type: file.type, + }; return new Promise(resolve => { transcriptResolve = resolve; }); } if (url === '/api/tts') { @@ -245,6 +255,7 @@ async function boot(scriptPath, reduced) { captures, recorders, analysers, assistantRows, toasts, reducedMotion, get sent() { return sent; }, get lastAudio() { return lastAudio; }, + get uploadedFile() { return uploadedFile; }, set now(value) { now = value; }, rejectCapture() { rejectNextCapture = true; }, resolveTranscript(payload) { @@ -261,7 +272,7 @@ async function boot(scriptPath, reduced) { }; } -async function normalMotionContract(scriptPath) { +async function normalMotionContract(scriptPath, mediaFixture) { const probe = await boot(scriptPath, false); const originalStyleLink = probe.styleLink; @@ -272,17 +283,38 @@ async function normalMotionContract(scriptPath) { await flush(); assert.equal(probe.captures.length, 1); + const mediaChunks = mediaFixture.chunks; + const preSpeechCount = mediaFixture.pre_speech_chunk_count; + for (const chunk of mediaChunks.slice(0, preSpeechCount)) { + probe.recorders[0].ondataavailable({ + data: new Blob([chunk], {type: mediaFixture.mime_type}), + }); + } + probe.analysers[0].level = 0.3; probe.runIntervals(); probe.runIntervals(); probe.runIntervals(); - probe.recorders[0].ondataavailable({data: new Blob(['speech'], {type: 'audio/webm'})}); + for (const chunk of mediaChunks.slice(preSpeechCount)) { + probe.recorders[0].ondataavailable({ + data: new Blob([chunk], {type: mediaFixture.mime_type}), + }); + } probe.now = 4000; probe.analysers[0].level = 0; probe.runIntervals(); await flush(); + await flush(); assert.match(probe.indicator.className, /\btranscribing\b/); assert.equal(probe.label.textContent, 'Transcribing…'); + assert.ok(probe.uploadedFile, 'transcription upload was not captured'); + assert.equal(probe.uploadedFile.name, 'voice-input.webm'); + assert.equal(probe.uploadedFile.type, mediaFixture.mime_type); + const expectedUpload = Buffer.concat([ + mediaChunks[0], + ...mediaChunks.slice(preSpeechCount - 3), + ]); + assert.deepEqual(Buffer.from(probe.uploadedFile.bytes), expectedUpload); probe.resolveTranscript({transcript: 'Hello Hermes'}); await flush(); @@ -310,8 +342,21 @@ async function normalMotionContract(scriptPath) { async function main() { const scriptPath = process.argv[2]; - assert.ok(scriptPath, 'usage: node hermes_voice_instrument_probe.js '); - const first = await normalMotionContract(scriptPath); + const fixturePath = process.argv[3]; + assert.ok( + scriptPath && fixturePath, + 'usage: node hermes_voice_instrument_probe.js ', + ); + const mediaFixture = JSON.parse(fs.readFileSync(fixturePath, 'utf8')); + const payload = fs.readFileSync(path.join(path.dirname(fixturePath), mediaFixture.payload_file)); + let offset = 0; + mediaFixture.chunks = mediaFixture.chunk_sizes.map(size => { + const chunk = payload.subarray(offset, offset + size); + offset += size; + return chunk; + }); + assert.equal(offset, payload.length, 'fixture chunk sizes do not cover the payload'); + const first = await normalMotionContract(scriptPath, mediaFixture); const {probe, originalStyleLink, capturesBeforeSpeech} = first; assert.equal(probe.captures.length, capturesBeforeSpeech); diff --git a/testing/tests/test_hermes_handsfree_stt.py b/testing/tests/test_hermes_handsfree_stt.py new file mode 100644 index 00000000..f0a11f4b --- /dev/null +++ b/testing/tests/test_hermes_handsfree_stt.py @@ -0,0 +1,203 @@ +"""Browser-to-ffmpeg contracts for Hermes hands-free transcription uploads.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[2] +VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js" +STT_PATCHER = ROOT / "dockerfiles/hermes-webui-stt-patch.py" +AGENT_FIXTURE = ROOT / "testing/fixtures/hermes-agent-9de9c25f" +MEDIA_FIXTURE = ROOT / "testing/fixtures/mediarecorder/chromium-webm-opus.json" +SAFE_CONVERSION_ERROR = ( + "Audio conversion failed: upload is invalid, incomplete, or uses an " + "unsupported codec" +) + + +def _media_chunks() -> tuple[dict[str, object], list[bytes]]: + fixture = json.loads(MEDIA_FIXTURE.read_text(encoding="utf-8")) + payload = MEDIA_FIXTURE.with_name(str(fixture["payload_file"])).read_bytes() + chunks = [] + offset = 0 + for size in fixture["chunk_sizes"]: + end = offset + int(size) + chunks.append(payload[offset:end]) + offset = end + assert offset == len(payload) + return fixture, chunks + + +def _late_speech_upload(*, preserve_header: bool) -> bytes: + fixture, chunks = _media_chunks() + count = int(fixture["pre_speech_chunk_count"]) + retained = chunks[count - 3 :] + if preserve_header: + retained.insert(0, chunks[0]) + return b"".join(retained) + + +def _patched_transcription_module(tmp_path: Path): + target = tmp_path / "hermes-agent" + shutil.copytree(AGENT_FIXTURE, target) + env = os.environ.copy() + env["HERMES_AGENT_PATCH_ROOT"] = str(target) + subprocess.run( + [sys.executable, str(STT_PATCHER)], + cwd=ROOT, + env=env, + check=True, + capture_output=True, + text=True, + ) + module_path = target / "tools/transcription_tools.py" + spec = importlib.util.spec_from_file_location( + "pinned_transcription_tools", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _convert(module, tmp_path: Path, payload: bytes, suffix: str): + source = tmp_path / f"voice-input{suffix}" + output_dir = tmp_path / f"converted-{suffix.removeprefix('.')}" + output_dir.mkdir() + source.write_bytes(payload) + return module._prepare_local_audio(str(source), str(output_dir)) + + +def test_fixture_reproduces_the_current_ebml_header_failure(tmp_path: Path): + """The red control is browser data after the old sliding pre-roll dropped chunk 0.""" + broken = tmp_path / "current-late-speech.webm" + broken.write_bytes(_late_speech_upload(preserve_header=False)) + + result = subprocess.run( + ["ffmpeg", "-v", "error", "-y", "-i", str(broken), str(tmp_path / "bad.wav")], + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + assert result.returncode != 0 + assert "EBML header" in result.stderr + + +def test_header_preserved_browser_webm_reaches_real_conversion_boundary( + tmp_path: Path, +): + module = _patched_transcription_module(tmp_path) + prepared, error = _convert( + module, + tmp_path, + _late_speech_upload(preserve_header=True), + ".webm", + ) + + assert error is None + assert prepared is not None + output = Path(prepared) + assert output.is_file() + assert output.read_bytes().startswith(b"RIFF") + + +def test_invalid_browser_upload_returns_bounded_error_without_ffmpeg_spam( + tmp_path: Path, +): + module = _patched_transcription_module(tmp_path) + prepared, error = _convert( + module, + tmp_path, + _late_speech_upload(preserve_header=False), + ".webm", + ) + + assert prepared is None + assert error == SAFE_CONVERSION_ERROR + assert len(error) < 128 + lowered = error.lower() + assert "ffmpeg version" not in lowered + assert "configuration:" not in lowered + assert "/tmp/" not in lowered + + +def test_local_conversion_accepts_browser_fallback_containers(tmp_path: Path): + module = _patched_transcription_module(tmp_path) + formats = ((".ogg", "libopus"), (".mp4", "aac")) + + for suffix, codec in formats: + source = tmp_path / f"source{suffix}" + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=0.4", + "-c:a", + codec, + str(source), + ], + check=True, + capture_output=True, + text=True, + timeout=15, + ) + prepared, error = _convert(module, tmp_path, source.read_bytes(), suffix) + assert error is None + assert prepared is not None + assert Path(prepared).read_bytes().startswith(b"RIFF") + + +def test_browser_uses_actual_recorder_mime_and_supported_extensions(): + source = VOICE_JS.read_text(encoding="utf-8") + + assert "let initialChunk=null" in source + assert "if(initialChunk){chunks.push(initialChunk);initialChunk=null;}" in source + assert "let recordedMime=recorder.mimeType||mime||''" in source + assert "if(event.data.type) recordedMime=event.data.type" in source + assert "audio/mp4;codecs=mp4a.40.2" in source + assert "if(normalized.indexOf('ogg')>=0) return 'ogg'" in source + assert "if(normalized.indexOf('mp4')>=0) return 'mp4'" in source + + +def test_stt_patch_is_built_fail_closed_into_the_webui_image(tmp_path: Path): + dockerfile = (ROOT / "dockerfiles/Dockerfile.hermes-webui").read_text( + encoding="utf-8" + ) + assert "COPY dockerfiles/hermes-webui-stt-patch.py" in dockerfile + assert "python /tmp/hermes-webui-stt-patch.py" in dockerfile + assert "/opt/hermes/tools/transcription_tools.py" in dockerfile + + drifted = tmp_path / "drifted-agent" + shutil.copytree(AGENT_FIXTURE, drifted) + transcription = drifted / "tools/transcription_tools.py" + source = transcription.read_text(encoding="utf-8") + transcription.write_text( + source.replace("Failed to convert audio for local STT", "upstream drift", 1), + encoding="utf-8", + ) + env = os.environ.copy() + env["HERMES_AGENT_PATCH_ROOT"] = str(drifted) + result = subprocess.run( + [sys.executable, str(STT_PATCHER)], + cwd=ROOT, + env=env, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "patch context changed" in result.stderr diff --git a/testing/tests/test_hermes_voice_instrument.py b/testing/tests/test_hermes_voice_instrument.py index eada0199..5b1a69c5 100644 --- a/testing/tests/test_hermes_voice_instrument.py +++ b/testing/tests/test_hermes_voice_instrument.py @@ -16,6 +16,9 @@ PATCHER = ROOT / "dockerfiles/hermes-webui-atlas-patch.py" VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js" VOICE_CSS = ROOT / "dockerfiles/hermes-webui-atlas-voice.css" DOM_PROBE = ROOT / "testing/probes/hermes_voice_instrument_probe.js" +MEDIARECORDER_FIXTURE = ( + ROOT / "testing/fixtures/mediarecorder/chromium-webm-opus.json" +) def _patched_fixture(tmp_path: Path) -> Path: @@ -86,7 +89,7 @@ def test_visual_states_have_distinct_layers_finite_error_and_reduced_motion(): def test_dom_probe_exercises_actual_injected_voice_script(): result = subprocess.run( - ["node", str(DOM_PROBE), str(VOICE_JS)], + ["node", str(DOM_PROBE), str(VOICE_JS), str(MEDIARECORDER_FIXTURE)], cwd=ROOT, check=True, capture_output=True,