From 09ba5f8ac09e3f05c829b7d12b8acc35af2fc3f3 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 11:21:35 -0300 Subject: [PATCH] hermes(voice): fix first-word clipping, always-stitch, panel glow, conversation mode Round 2 from live testing: - Clipping root cause: the endpointer committed on any 1.1s pause, so a thinking pause after a sentence opener sent one word; a speculative Whisper pass over that fragment then stalled the real commit ~3.5s on the Jetson. Young utterances now hold a 1.8s endpoint until 1.2s of speech accrues, speculative decode waits for 700ms of speech, resume is unconditional after any 650ms gap (server-frozen snapshots can never reach commit), and the noise floor is capped so playback echo cannot deafen onset. Deterministic capture-continuity probe added. - Stitching now fires for any barge-cancelled send within 20s, regardless of partial assistant output. - The workspace-drawer dark rectangle was our own HUX chrome resolving undefined theme tokens (--bg-primary) to a flat box; bootstrap.css bridges the real theme tokens and drops a compositor-hazard backdrop-filter. - Conversation mode: full-viewport hands-free overlay with an energy- driven orb (mic RMS in, TTS activity out), state caption, live transcript/reply captions, mute and exit controls, focus trap, Escape, reduced-motion support. Voice lane 253 tests green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- dockerfiles/hermes-webui-atlas-voice.css | 234 ++++++ dockerfiles/hermes-webui-atlas-voice.js | 209 ++++- dockerfiles/hermes-webui-hux/bootstrap.css | 25 + testing/probes/hermes_voice_capture_probe.js | 786 ++++++++++++++++++ .../test_hermes_voice_capture_continuity.py | 158 ++++ testing/tests/test_hermes_voice_instrument.py | 5 +- 6 files changed, 1405 insertions(+), 12 deletions(-) create mode 100644 testing/probes/hermes_voice_capture_probe.js create mode 100644 testing/tests/test_hermes_voice_capture_continuity.py diff --git a/dockerfiles/hermes-webui-atlas-voice.css b/dockerfiles/hermes-webui-atlas-voice.css index 910fbfad..b587a124 100644 --- a/dockerfiles/hermes-webui-atlas-voice.css +++ b/dockerfiles/hermes-webui-atlas-voice.css @@ -334,3 +334,237 @@ transform: scale(1.03); } } + +/* ── Conversation mode overlay ─────────────────────────────────────────── */ +/* Full-viewport hands-free conversation surface. Created lazily by + atlas-voice.js on activation and removed on deactivate. The orb breathes in + idle listening, follows live microphone energy through + --conversation-level, and pulses while Hermes speaks (.is-playing, driven + from TTS playback activity). State accents reuse the indicator palette. */ +.voice-conversation { + --voice-accent: 69, 218, 207; + --voice-accent-secondary: 73, 169, 209; + --conversation-level: 0; + position: fixed; + inset: 0; + z-index: 420; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 24px; + padding: 34px 22px calc(28px + env(safe-area-inset-bottom, 0px)); + background: + radial-gradient(circle at 50% 30%, rgba(var(--voice-accent), 0.11), transparent 52%), + radial-gradient(circle at 22% 92%, rgba(157, 142, 224, 0.07), transparent 44%), + linear-gradient(160deg, #0b1120, #070a12 55%, #081018); + color: rgba(240, 247, 255, 0.94); +} + +.voice-conversation[data-voice-state="transcribing"] { + --voice-accent: 73, 193, 218; + --voice-accent-secondary: 93, 129, 224; +} + +.voice-conversation[data-voice-state="thinking"] { + --voice-accent: 135, 111, 244; + --voice-accent-secondary: 81, 62, 190; +} + +.voice-conversation[data-voice-state="speaking"] { + --voice-accent: 249, 169, 105; + --voice-accent-secondary: 236, 112, 137; +} + +.voice-conversation[data-voice-state="error"] { + --voice-accent: 235, 135, 88; + --voice-accent-secondary: 198, 74, 78; +} + +.voice-conversation-orb { + position: relative; + width: min(46vmin, 220px); + height: min(46vmin, 220px); + flex: none; +} + +.voice-conversation-orb-halo, +.voice-conversation-orb-core, +.voice-conversation-orb-ring { + position: absolute; + inset: 0; + display: block; + border-radius: 50%; + pointer-events: none; +} + +.voice-conversation-orb-halo { + background: radial-gradient(circle, rgba(var(--voice-accent), 0.34), rgba(var(--voice-accent-secondary), 0.1) 52%, transparent 74%); + filter: blur(10px); + opacity: 0.75; + transform: scale(calc(1 + (var(--conversation-level) * 0.3))); + transition: transform 130ms ease-out; +} + +.voice-conversation-orb-core { + inset: 12%; + background: + radial-gradient(circle at 36% 28%, rgba(255, 255, 255, 0.34), transparent 34%), + radial-gradient(circle at 52% 68%, rgba(var(--voice-accent), 0.62), rgba(var(--voice-accent-secondary), 0.4) 62%, transparent 86%), + linear-gradient(150deg, rgba(34, 44, 63, 0.95), rgba(9, 13, 24, 0.98)); + box-shadow: + inset 0 1px 2px rgba(255, 255, 255, 0.28), + 0 10px 42px rgba(0, 0, 0, 0.45), + 0 0 44px rgba(var(--voice-accent), 0.3); + transform: scale(calc(1 + (var(--conversation-level) * 0.14))); + transition: transform 110ms ease-out; +} + +.voice-conversation-orb-ring { + inset: 3%; + border: 1.5px solid rgba(var(--voice-accent), 0.4); + box-shadow: 0 0 22px rgba(var(--voice-accent), 0.16); + opacity: 0.85; +} + +/* Idle listening: slow breathing. */ +.voice-conversation[data-voice-state="listening"] .voice-conversation-orb-halo { + animation: voice-conversation-breathe 4.4s ease-in-out infinite; +} + +/* Transcribing/thinking: the ring sweeps. */ +.voice-conversation[data-voice-state="transcribing"] .voice-conversation-orb-ring, +.voice-conversation[data-voice-state="thinking"] .voice-conversation-orb-ring { + border-top-color: rgba(var(--voice-accent), 0.95); + border-right-color: rgba(var(--voice-accent-secondary), 0.4); + animation: voice-conversation-sweep 2.6s linear infinite; +} + +/* Hermes speaking: the halo waves outward while playback is live. */ +.voice-conversation.is-playing .voice-conversation-orb-halo { + animation: voice-conversation-wave 1.9s ease-out infinite; +} + +.voice-conversation.is-playing .voice-conversation-orb-core { + animation: voice-conversation-swell 1.9s ease-in-out infinite; +} + +.voice-conversation-state { + min-height: 20px; + font-size: 15px; + font-weight: 650; + letter-spacing: 0.02em; + text-align: center; + color: rgba(238, 246, 255, 0.9); +} + +.voice-conversation-captions { + display: flex; + flex-direction: column; + gap: 10px; + width: min(100%, 40rem); + max-height: 30vh; + overflow: hidden; + text-align: center; +} + +.voice-conversation-caption-user { + margin: 0; + font-size: 14px; + line-height: 1.5; + color: rgba(178, 196, 214, 0.88); + font-style: italic; +} + +.voice-conversation-caption-assistant { + margin: 0; + font-size: 16px; + line-height: 1.6; + color: rgba(240, 247, 255, 0.95); +} + +.voice-conversation-controls { + display: flex; + gap: 14px; + margin-top: 6px; +} + +.voice-conversation-controls button { + font: inherit; + font-size: 13px; + font-weight: 600; + color: rgba(233, 243, 252, 0.92); + background: rgba(255, 255, 255, 0.07); + border: 1px solid rgba(var(--voice-accent), 0.36); + border-radius: 999px; + padding: 10px 18px; + cursor: pointer; +} + +.voice-conversation-controls button:hover { + background: rgba(255, 255, 255, 0.12); +} + +.voice-conversation-controls button:focus-visible { + outline: 2px solid rgb(var(--voice-accent)); + outline-offset: 2px; +} + +.voice-conversation.is-muted .voice-conversation-mute { + color: #ffd9c7; + border-color: rgba(235, 135, 88, 0.6); + background: rgba(235, 135, 88, 0.14); +} + +.voice-conversation.is-muted .voice-conversation-orb-halo, +.voice-conversation.is-muted .voice-conversation-orb-core { + animation: none; + transform: none; + filter: blur(10px) saturate(0.4); +} + +.voice-conversation.is-muted .voice-conversation-orb-core { + filter: saturate(0.35) brightness(0.8); +} + +@keyframes voice-conversation-breathe { + 0%, 100% { opacity: 0.55; transform: scale(calc(0.96 + (var(--conversation-level) * 0.3))); } + 50% { opacity: 0.85; transform: scale(calc(1.05 + (var(--conversation-level) * 0.3))); } +} + +@keyframes voice-conversation-sweep { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +@keyframes voice-conversation-wave { + 0% { opacity: 0.8; transform: scale(0.96); } + 55% { opacity: 0.5; transform: scale(1.12); } + 100% { opacity: 0.75; transform: scale(0.98); } +} + +@keyframes voice-conversation-swell { + 0%, 100% { transform: scale(0.99); } + 50% { transform: scale(1.05); } +} + +@media (max-width: 640px) { + .voice-conversation { gap: 18px; } + .voice-conversation-captions { max-height: 34vh; } +} + +/* Reduced motion: static orb states — the accent color and the state caption + still communicate every phase without animation or level-driven motion. */ +@media (prefers-reduced-motion: reduce) { + .voice-conversation *, + .voice-conversation *::before, + .voice-conversation *::after { + animation: none !important; + transition: none !important; + } + + .voice-conversation-orb-halo, + .voice-conversation-orb-core { + transform: none !important; + } +} diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index ac694a1f..fed8ffc1 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -72,6 +72,23 @@ // A response cancelled by barge-in leaves the user mid-thought: the next // finalized utterance inside this window is sent as one stitched message. const STITCH_WINDOW_MS=20000; + // Endpointing. A short utterance ("The…") is usually a sentence still being + // formed, so while less than VAD_COMMITTED_SPEECH_MS of voiced audio has + // been collected the endpoint silence window stretches to + // VAD_EARLY_SILENCE_MS: a thinking pause right after the first word can no + // longer commit a one-word fragment of the sentence. + const VAD_EARLY_SILENCE_MS=1800; + const VAD_COMMITTED_SPEECH_MS=1200; + // The private STT server freezes a speculative end-of-speech snapshot after + // 650ms of server-side silence even when this client never sent speculate. + // Speech resuming after a gap that long always sends an explicit resume so + // a frozen first-word snapshot can never survive into the commit. + const SERVER_EOS_SILENCE_MS=650; + // Never speculate on the first inter-word gap of a young utterance: a + // wasted Whisper pass over one word occupies the Jetson for seconds and + // stalls the real commit that follows (observed live as multi-second + // commit pending waits). + const SPECULATE_MIN_SPEECH_MS=700; const TTS_SPEED_DEFAULT=1.15; const TTS_SPEED_MIN=0.5; const TTS_SPEED_MAX=2; @@ -133,6 +150,7 @@ label.textContent=customLabel||STATE_LABELS[next]||''; bar.style.display=(active&&next!=='idle')||next==='error'?'':'none'; resetInputLevel(); + syncConversationOverlay(next,customLabel); } function resetInputLevel(){ @@ -147,6 +165,146 @@ visualInputLevel=(visualInputLevel*0.72)+(target*0.28); indicator.style.setProperty('--voice-ripple-scale',(1.035+(visualInputLevel*0.16)).toFixed(3)); indicator.style.setProperty('--voice-ripple-opacity',(0.26+(visualInputLevel*0.48)).toFixed(3)); + if(conversation) conversation.root.style.setProperty('--conversation-level',visualInputLevel.toFixed(3)); + } + + // ── Conversation mode overlay ───────────────────────────────────────── + // Hands-free is a full-screen conversation: a breathing orb that follows + // microphone energy while listening and TTS playback while speaking, the + // live state caption, streaming user/assistant captions, and mute/exit + // controls. The overlay is created lazily on activation, removed completely + // on deactivate, keeps no storage, and is silently skipped in environments + // without a usable DOM (headless contract probes). + let conversation=null; + + function conversationUsable(){ + return !!(typeof document!=='undefined'&&document&&document.body&&typeof document.createElement==='function'); + } + + function conversationNode(tag,className,attributes){ + const node=document.createElement(tag); + if(className) node.className=className; + if(attributes) Object.keys(attributes).forEach(function(name){node.setAttribute(name,attributes[name]);}); + return node; + } + + function syncConversationOverlay(next,customLabel){ + if(!conversation) return; + conversation.root.dataset.voiceState=next; + conversation.stateEl.textContent=customLabel||STATE_LABELS[next]||''; + } + + function setConversationUserCaption(text){ + if(conversation) conversation.userCaption.textContent=String(text||'').slice(-600); + } + + function setConversationAssistantCaption(text){ + if(conversation) conversation.assistantCaption.textContent=String(text||'').slice(-900); + } + + function setConversationPlaying(playing){ + if(!conversation) return; + if(playing) conversation.root.classList.add('is-playing'); + else conversation.root.classList.remove('is-playing'); + } + + function applyConversationMute(){ + // True microphone mute on the session's single retained getUserMedia + // stream: disabled tracks deliver silence to the recorder, the VAD and + // the streaming STT worklet alike. No second capture is ever opened. + if(!conversation) return; + try{ + const tracks=stream&&stream.getAudioTracks?stream.getAudioTracks():[]; + tracks.forEach(function(track){track.enabled=!conversation.muted;}); + }catch(_){ } + } + + function setConversationMuted(muted){ + if(!conversation) return; + conversation.muted=muted; + applyConversationMute(); + try{ + conversation.muteBtn.setAttribute('aria-pressed',muted?'true':'false'); + conversation.muteBtn.textContent=muted?'Unmute microphone':'Mute microphone'; + if(muted) conversation.root.classList.add('is-muted'); + else conversation.root.classList.remove('is-muted'); + }catch(_){ } + } + + function conversationKeydown(event){ + if(!conversation) return; + if(event.key==='Escape'){ + if(event.preventDefault) event.preventDefault(); + deactivate(true); + return; + } + if(event.key==='Tab'){ + // Minimal focus trap across the two overlay controls. + const stops=[conversation.muteBtn,conversation.exitBtn]; + const current=stops.indexOf(document.activeElement); + const index=current<0?(event.shiftKey?0:stops.length-1):current; + const next=stops[(index+(event.shiftKey?stops.length-1:1))%stops.length]; + if(event.preventDefault) event.preventDefault(); + if(next&&next.focus) next.focus(); + } + } + + function openConversationOverlay(){ + if(conversation||!conversationUsable()) return; + try{ + const root=conversationNode('div','voice-conversation',{ + role:'dialog','aria-modal':'true','aria-label':'Voice conversation',tabindex:'-1', + }); + const orb=conversationNode('div','voice-conversation-orb',{'aria-hidden':'true'}); + orb.appendChild(conversationNode('span','voice-conversation-orb-halo')); + orb.appendChild(conversationNode('span','voice-conversation-orb-core')); + orb.appendChild(conversationNode('span','voice-conversation-orb-ring')); + const stateEl=conversationNode('div','voice-conversation-state',{'aria-hidden':'true'}); + const captions=conversationNode('div','voice-conversation-captions',{'aria-live':'polite'}); + const userCaption=conversationNode('p','voice-conversation-caption-user'); + const assistantCaption=conversationNode('p','voice-conversation-caption-assistant'); + captions.appendChild(userCaption); + captions.appendChild(assistantCaption); + const controls=conversationNode('div','voice-conversation-controls'); + const muteBtn=conversationNode('button','voice-conversation-mute',{type:'button','aria-pressed':'false'}); + muteBtn.textContent='Mute microphone'; + const exitBtn=conversationNode('button','voice-conversation-exit',{type:'button'}); + exitBtn.textContent='Exit voice mode'; + controls.appendChild(muteBtn); + controls.appendChild(exitBtn); + root.appendChild(orb); + root.appendChild(stateEl); + root.appendChild(captions); + root.appendChild(controls); + muteBtn.addEventListener('click',function(){if(conversation) setConversationMuted(!conversation.muted);}); + exitBtn.addEventListener('click',function(){deactivate(true);}); + root.addEventListener('keydown',conversationKeydown); + document.body.appendChild(root); + conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,muted:false}; + syncConversationOverlay(state); + if(root.focus) root.focus(); + }catch(_){ + // A partially built overlay is never attached: document.body.appendChild + // is the last DOM mutation above, so a stub DOM simply keeps the + // compact voice bar. + conversation=null; + } + } + + function removeConversationOverlay(){ + const overlay=conversation; + conversation=null; + if(!overlay) return; + try{ + if(overlay.root.removeEventListener) overlay.root.removeEventListener('keydown',conversationKeydown); + if(overlay.root.parentNode) overlay.root.parentNode.removeChild(overlay.root); + }catch(_){ } + // The session's tracks are always left enabled on the way out; a real + // deactivation stops them entirely via releaseMicrophone(). + try{ + const tracks=stream&&stream.getAudioTracks?stream.getAudioTracks():[]; + tracks.forEach(function(track){track.enabled=true;}); + }catch(_){ } } function clearErrorTimer(){ @@ -329,6 +487,7 @@ cancelSpeechTurn(); releaseMicrophone(); stopPlayback(); + removeConversationOverlay(); modeBtn.classList.remove('active'); setState('error',message); clearErrorTimer(); @@ -409,6 +568,7 @@ thinkingSession=null; thinkingTurnId=''; clearErrorTimer(); + removeConversationOverlay(); clearSttLanguage(); suppressAutoRead=false; pendingStitch=null; @@ -472,15 +632,16 @@ } function recordPendingStitch(){ - // Called at the moment a model turn is cancelled by user speech. Stitching - // only applies when the cancelled turn produced NO visible assistant text: - // the model never answered, so the next utterance restates the complete - // thought as one message. Any partial answer means the interruption stands - // on its own and the next utterance is sent alone. + // Called at the moment a model turn is cancelled by user speech. The + // interrupted send left the user mid-thought regardless of how much of an + // answer had already streamed — responses start streaming almost + // immediately, so gating on visible output made stitching almost never + // fire. The next finalized utterance inside STITCH_WINDOW_MS is therefore + // always sent together with the interrupted transcript as one message. + // Normal completion, deactivation and window expiry still clear it. const sent=lastSentTranscript; lastSentTranscript=null; if(!sent||!sent.text) return; - if(currentAssistantText()){pendingStitch=null;return;} pendingStitch={text:sent.text,at:Date.now()}; } @@ -503,6 +664,8 @@ pendingStitch=null; composer.value=stitch?stitch.text+' '+text:text; if(typeof window.autoResize==='function') window.autoResize(); + setConversationUserCaption(composer.value); + setConversationAssistantCaption(''); setState('thinking'); // The live VAD/STT capture is the barge-in detector while it runs; the // energy-only monitor remains only as a fallback when capture is down. @@ -717,6 +880,7 @@ if(visible&&active&&captureTurnId===turnId&&state==='listening'){ const preview=visible.length>72?visible.slice(0,69)+'…':visible; label.textContent='Listening · '+preview+(stable?'':' · provisional'); + setConversationUserCaption(visible); if(stable!==lastPreflightText){ if(stable){ lastPreflightText=stable; @@ -769,7 +933,11 @@ sendJson({type:'speculate',turn_id:turnId}); }, resume:function(){ - if(settled||committed||!speculative) return; + // Deliberately not gated on a prior speculate() from this client: the + // server also freezes an EOS snapshot on its own silence detector, so + // resume must always reach the wire when speech restarts. The server + // treats a redundant resume as a harmless epoch bump. + if(settled||committed) return; speculative=false; sendJson({type:'resume',turn_id:turnId}); }, @@ -1087,6 +1255,7 @@ return; } stream=capture; + applyConversationMute(); const captureAec=captureAecIsUsable(stream); const Context=window.AudioContext||window.webkitAudioContext; if(reusedCapture&&reusedCapture.context){ @@ -1133,6 +1302,9 @@ const chunks=[]; let heardSpeech=!!(reusedCapture&&reusedCapture.heardSpeech); let voiceFrames=heardSpeech?3:0; + // Voiced milliseconds collected this utterance. A barge handoff already + // carries confirmed speech, so it starts past the young-utterance hold. + let speechMs=heardSpeech?VAD_COMMITTED_SPEECH_MS:0; let noiseFloor=0.008; let lastSpeech=Date.now(); let speculative=false; @@ -1222,10 +1394,16 @@ bargeArmAt=now+100; } playbackWasLive=playbackLive; + setConversationPlaying(playbackLive); const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006); const voiceNow=rms>speechThreshold&&(!playbackLive||(captureAec&&now>=bargeArmAt)); - if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);} + // Adapt the floor only toward energies at or below the current + // threshold (like the fallback energy monitor): echo residue during + // continuous capture must never ratchet the threshold above real + // speech and deafen onset detection for the next utterance. + if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(Math.min(rms,speechThreshold)*0.06);} voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0); + if(voiceNow) speechMs+=100; if(playbackLive){ if(voiceFrames>=BARGE_DUCK_FRAMES&&!ducked){ducked=true;setPlaybackDucked(true);} if(!voiceFrames&&ducked){ducked=false;setPlaybackDucked(false);} @@ -1243,14 +1421,21 @@ // cancel the response side only; this capture keeps running. if(state==='thinking'||state==='speaking') bargeFromLiveCapture(); }else if(heardSpeech&&voiceNow){ + const speechGapMs=now-lastSpeech; lastSpeech=now; - if(speculative&&streamingStt){streamingStt.resume();speculative=false;} + // Resume after our own speculate AND after any gap long enough for + // the server's independent 650ms end-of-speech detector: either + // party may have frozen a partial-utterance snapshot by now. + if((speculative||speechGapMs>=SERVER_EOS_SILENCE_MS)&&streamingStt){streamingStt.resume();speculative=false;} } - if(heardSpeech&&!voiceNow&&!speculative&&(now-lastSpeech)>=speculateMs&&streamingStt){ + if(heardSpeech&&!voiceNow&&!speculative&&speechMs>=SPECULATE_MIN_SPEECH_MS&&(now-lastSpeech)>=speculateMs&&streamingStt){ streamingStt.speculate(); speculative=true; } - const finished=heardSpeech&&(now-lastSpeech)>=silenceMs; + // A young utterance holds its endpoint longer: pausing to think right + // after the first word must not send a one-word fragment. + const endpointSilenceMs=speechMs=endpointSilenceMs; const timedOut=now-started>=90000; const idle=(!heardSpeech)&&(now-started)>=20000; if(finished||timedOut||idle){ @@ -1829,6 +2014,7 @@ if(isFinal){thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();cancelThinkingCues();restartSoon(token,250);} return; } + setConversationAssistantCaption(text); // Any answer text, even a not-yet-speakable partial clause, owns the audio // timeline from this point forward. A cue must never overlap or become part // of the answer queue. @@ -1915,6 +2101,7 @@ clearSttLanguage(); modeBtn.classList.add('active'); toast('Hands-free private voice mode on'); + openConversationOverlay(); if(typeof window.stopTTS==='function') window.stopTTS(); if(typeof S!=='undefined'&&S.busy){setState('thinking');return;} startListening(token); diff --git a/dockerfiles/hermes-webui-hux/bootstrap.css b/dockerfiles/hermes-webui-hux/bootstrap.css index f4258935..4e087a10 100644 --- a/dockerfiles/hermes-webui-hux/bootstrap.css +++ b/dockerfiles/hermes-webui-hux/bootstrap.css @@ -94,3 +94,28 @@ border-color: CanvasText; } } + +/* ── Theme bridge for the Workspace chrome ─────────────────────────────── */ +/* The pinned upstream WebUI theme (and hermes-brand.css on top of it) defines + --bg/--text but never --bg-primary/--text-primary, so every color-mix() + surface above resolved its fallback constant (#0a101b): with the drawer + open that painted a large flat dark rectangle that visibly broke the page's + themed background. Bridge the tokens to the app theme so the drawer, its + toggle and the shell inherit the real page background (light and dark) — + layout and component styling are deliberately untouched. */ +.hux-workspace-toggle, +.hux-workspace-drawer, +.hux-shell { + --bg-primary: var(--bg, #0a101b); + --text-primary: var(--text, #e7eef6); +} + +/* The header previously painted a hard-coded dark wash and sampled the page + through backdrop-filter; blurring the backdrop of a sticky header inside a + fixed overflow scroller is also a known Chromium compositor hazard on this + app's gradient background (see the stale-tile note in hermes-brand.css). + Track the theme instead and let the opaque drawer carry the surface. */ +.hux-workspace-drawer__header { + backdrop-filter: none; + background: color-mix(in srgb, var(--bg, #0a101b) 92%, transparent); +} diff --git a/testing/probes/hermes_voice_capture_probe.js b/testing/probes/hermes_voice_capture_probe.js new file mode 100644 index 00000000..c491fae9 --- /dev/null +++ b/testing/probes/hermes_voice_capture_probe.js @@ -0,0 +1,786 @@ +// Deterministic continuous-capture probe for dockerfiles/hermes-webui-atlas-voice.js. +// +// Drives the real script through complete streaming hands-free turns against a +// stub STT WebSocket whose VAD, speculative EOS-freeze, resume and commit +// semantics faithfully mirror dockerfiles/hermes-jetson-stt-server.py. Words +// are encoded as distinct constant PCM amplitudes so the "transcript" of any +// audio snapshot is exactly the ordered set of words whose samples survived +// endpointing — a first-word-only regression is directly observable as a +// truncated transcript in the sent composer text. +// +// node hermes_voice_capture_probe.js +// +// Prints one JSON object per scenario to stdout; the python wrapper asserts. +'use strict'; + +const fs = require('fs'); +const vm = require('vm'); + +const SCRIPT_PATH = process.argv[2]; +if (!SCRIPT_PATH) { + throw new Error('usage: hermes_voice_capture_probe.js '); +} +const SOURCE = fs.readFileSync(SCRIPT_PATH, 'utf8'); + +class FakeURL extends URL {} +FakeURL.createObjectURL = () => 'blob:capture-probe'; +FakeURL.revokeObjectURL = () => {}; + +const SAMPLE_RATE = 16000; +const TICK_MS = 100; +const TICK_SAMPLES = (SAMPLE_RATE * TICK_MS) / 1000; + +// Server constants mirrored from hermes-jetson-stt-server.py. +const SERVER_END_SILENCE_SAMPLES = (SAMPLE_RATE * 650) / 1000; +const SERVER_TAIL_SAMPLES = (SAMPLE_RATE * 220) / 1000; + +function flush() { + return new Promise((resolve) => { + let hops = 0; + (function hop() { + hops += 1; + if (hops > 16) { resolve(); return; } + setImmediate(hop); + })(); + }); +} + +function makeElement(id) { + const element = { + id, + value: '', + textContent: '', + className: '', + dataset: {}, + attributes: {}, + children: [], + style: { + values: new Map(), + display: '', + setProperty(name, value) { this.values.set(name, String(value)); }, + getPropertyValue(name) { return this.values.get(name) || ''; }, + removeProperty(name) { this.values.delete(name); }, + }, + listeners: [], + setAttribute(name, value) { this.attributes[name] = String(value); }, + getAttribute(name) { + return Object.prototype.hasOwnProperty.call(this.attributes, name) + ? this.attributes[name] : null; + }, + addEventListener(type, handler) { this.listeners.push({ type, handler }); }, + removeEventListener(type, handler) { + element.listeners = element.listeners.filter((entry) => entry.handler !== handler); + }, + appendChild(child) { element.children.push(child); child.parentNode = element; return child; }, + removeChild(child) { + element.children = element.children.filter((entry) => entry !== child); + child.parentNode = null; + return child; + }, + contains() { return false; }, + focus() {}, + querySelector() { return null; }, + querySelectorAll() { return []; }, + insertBefore() {}, + click() { + const event = { preventDefault() {}, stopImmediatePropagation() {} }; + element.listeners + .filter((entry) => entry.type === 'click') + .forEach((entry) => entry.handler(event)); + }, + }; + element.classList = { + add(name) { + const names = new Set(element.className.split(/\s+/).filter(Boolean)); + names.add(name); + element.className = [...names].join(' '); + }, + remove(name) { + element.className = element.className + .split(/\s+/).filter((value) => value && value !== name).join(' '); + }, + contains(name) { return element.className.split(/\s+/).includes(name); }, + toggle(name, force) { + const present = this.contains(name); + const next = force === undefined ? !present : !!force; + if (next) this.add(name); else this.remove(name); + return next; + }, + }; + return element; +} + +// Faithful port of StreamingTranscription's endpointing/speculation contract. +class StubSttServer { + constructor(words) { + // words: {amplitude(int16) -> word string} + this.words = words; + this.chunks = []; // Int16Array chunks in arrival order + this.totalSamples = 0; + this.noiseFloor = 0.004; + this.heardSpeech = false; + this.atEos = false; + this.silenceSamples = 0; + this.lastSpeechSample = 0; + this.clientActive = false; + this.epoch = 0; + this.frozenSnapshotSamples = -1; + this.committed = false; + this.events = []; + } + + snapshotSamples() { + if (this.atEos && this.frozenSnapshotSamples >= 0) return this.frozenSnapshotSamples; + let cutoff = this.totalSamples; + if (this.heardSpeech && this.lastSpeechSample) { + cutoff = Math.min(cutoff, this.lastSpeechSample + SERVER_TAIL_SAMPLES); + } + return cutoff; + } + + append(int16) { + if (this.committed) throw new Error('server: audio after commit'); + this.chunks.push(int16); + this.totalSamples += int16.length; + let energy = 0; + for (let index = 0; index < int16.length; index += 1) { + const value = int16[index] / 32768; + energy += value * value; + } + const rms = int16.length ? Math.sqrt(energy / int16.length) : 0; + const threshold = Math.max(0.012, this.noiseFloor * 2.5 + 0.004); + const speech = this.clientActive || rms >= threshold; + if (!this.heardSpeech && !speech) { + this.noiseFloor = this.noiseFloor * 0.96 + rms * 0.04; + } + if (speech) { + if (this.atEos) { + this.epoch += 1; + this.frozenSnapshotSamples = -1; + this.events.push('recover-by-rms'); + } + this.heardSpeech = true; + this.atEos = false; + this.silenceSamples = 0; + this.lastSpeechSample = this.totalSamples; + } else if (this.heardSpeech) { + this.silenceSamples += int16.length; + if (this.silenceSamples >= SERVER_END_SILENCE_SAMPLES && !this.atEos) { + this.atEos = true; + this.frozenSnapshotSamples = this.snapshotSamples(); + this.events.push('auto-eos-freeze'); + } + } + } + + speculate() { + this.clientActive = false; + this.atEos = true; + this.frozenSnapshotSamples = this.snapshotSamples(); + this.events.push('speculate-freeze'); + } + + resume() { + this.atEos = false; + this.silenceSamples = 0; + this.epoch += 1; + this.clientActive = true; + this.frozenSnapshotSamples = -1; + this.events.push('resume-clear'); + } + + transcript() { + const cutoff = this.snapshotSamples(); + const decoded = []; + let position = 0; + let currentWord = null; + let runLength = 0; + const flushRun = () => { + // Ignore sub-40ms runs: resampler transition samples, not words. + if (currentWord && runLength >= SAMPLE_RATE * 0.04) decoded.push(currentWord); + currentWord = null; + runLength = 0; + }; + for (const chunk of this.chunks) { + for (let index = 0; index < chunk.length; index += 1) { + if (position >= cutoff) { flushRun(); return decoded.join(' '); } + position += 1; + const magnitude = Math.abs(chunk[index]); + let word = null; + for (const [amplitude, name] of Object.entries(this.words)) { + if (Math.abs(magnitude - Number(amplitude)) <= 200) { word = name; break; } + } + if (word === currentWord) { + runLength += 1; + } else { + flushRun(); + currentWord = word; + runLength = word ? 1 : 0; + } + } + } + flushRun(); + return decoded.join(' '); + } +} + +function makeHarness(options = {}) { + const clock = { now: 1000000 }; + const timeouts = []; + const intervals = new Map(); + let timerId = 1; + + const sends = []; + const toasts = []; + const transcribeUploads = []; + const servers = []; + const captureNodes = []; + const analysers = []; + const recorders = []; + let assistantRows = []; + let streamCounter = 0; + + const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' }; + const micTracks = []; + + const elements = {}; + ['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg'] + .forEach((id) => { elements[id] = makeElement(id); }); + const bodyElement = makeElement('body'); + + // ── Microphone model ──────────────────────────────────────────────────── + // Per tick the mic has one (analyser level, pcm amplitude) pair. The level + // feeds the script's VAD analyser; the amplitude becomes the PCM the + // capture worklet delivers to the streaming session. + const mic = { level: 0, amplitude: 0 }; + + class FakeAnalyser { + constructor() { this.fftSize = 0; analysers.push(this); } + getByteTimeDomainData(samples) { + for (let index = 0; index < samples.length; index += 1) { + const offset = Math.round(mic.level * 127); + samples[index] = 128 + (index % 2 ? offset : -offset); + } + } + disconnect() {} + } + + class FakeAudioContext { + constructor() { + this.sampleRate = SAMPLE_RATE; + this.destination = {}; + this.audioWorklet = { addModule: async () => {} }; + this.closed = false; + } + createAnalyser() { return new FakeAnalyser(); } + createBiquadFilter() { + return { type: '', frequency: { value: 0 }, Q: { value: 0 }, connect() {}, disconnect() {} }; + } + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; } + resume() { return Promise.resolve(); } + close() { this.closed = true; return Promise.resolve(); } + } + + class FakeAudioWorkletNode { + constructor(_context, name) { + this.name = name; + this.connected = true; + const node = this; + this.port = { + onmessage: null, + postMessage(message) { + if ((message || {}).type === 'flush') { + queueMicrotask(() => { + if (node.port.onmessage) node.port.onmessage({ data: { type: 'flushed' } }); + }); + } + }, + }; + if (name === 'atlas-pcm-capture') captureNodes.push(this); + } + connect() {} + disconnect() { this.connected = false; } + } + + class FakeMediaRecorder { + static isTypeSupported() { return true; } + constructor(stream, recorderOptions) { + this.stream = stream; + this.state = 'inactive'; + this.mimeType = (recorderOptions && recorderOptions.mimeType) || 'audio/webm;codecs=opus'; + this.ondataavailable = null; + this.onstop = null; + recorders.push(this); + } + start() { this.state = 'recording'; } + stop() { + if (this.state === 'inactive') return; + this.state = 'inactive'; + if (this.ondataavailable) { + this.ondataavailable({ data: { size: 512, type: 'audio/webm;codecs=opus' } }); + } + const recorder = this; + // Real recorders finalize asynchronously. + queueMicrotask(() => { if (recorder.onstop) recorder.onstop(); }); + } + } + + class FakeWebSocket { + constructor(url, protocols) { + this.url = url; + this.protocols = protocols; + this.readyState = 0; + this.bufferedAmount = 0; + this.binaryType = ''; + this.onopen = null; + this.onmessage = null; + this.onerror = null; + this.onclose = null; + this.server = new StubSttServer(words); + this.turnId = ''; + servers.push(this); + const socket = this; + queueMicrotask(() => { + if (socket.readyState !== 0) return; + socket.readyState = 1; + if (socket.onopen) socket.onopen(); + }); + } + deliver(payload) { + const socket = this; + queueMicrotask(() => { + if (socket.readyState !== 1) return; + if (socket.onmessage) socket.onmessage({ data: JSON.stringify(payload) }); + }); + } + send(data) { + if (this.readyState !== 1) throw new Error('socket not open'); + if (typeof data === 'string') { + const message = JSON.parse(data); + if (message.type === 'start') { this.turnId = message.turn_id; return; } + if (message.type === 'speculate') { this.server.speculate(); return; } + if (message.type === 'resume') { this.server.resume(); return; } + if (message.type === 'cancel') { this.server.events.push('cancel'); return; } + if (message.type === 'commit') { + this.server.committed = true; + this.deliver({ + type: 'final', + turn_id: this.turnId, + transcript: this.server.transcript(), + language: 'en', + }); + } + return; + } + this.server.append(new Int16Array(data)); + } + close(code) { this.readyState = 3; this.closeCode = code; } + } + + async function fetchStub(url, init) { + if (url === '/api/transcribe/capability') { + return { ok: true, status: 200, json: async () => ({ available: true, provider: 'local_command' }) }; + } + if (url === '/api/voice/streaming/capability') { + return { + ok: true, + status: 200, + json: async () => ({ + tts: { available: false }, + stt: { available: true, transport: 'websocket', format: 'pcm_s16le', sample_rate: 16000 }, + preflight: { available: false }, + }), + }; + } + if (url === '/api/transcribe') { + transcribeUploads.push({ body: init && init.body }); + return { ok: true, status: 200, json: async () => ({ transcript: 'CONTAINER-FALLBACK', language: 'en' }) }; + } + if (url === '/api/tts') { + return { ok: true, status: 200, blob: async () => ({ synthetic: true }), json: async () => ({}) }; + } + if (String(url).indexOf('api/chat/cancel') >= 0) { + context.S.busy = false; + context.S.activeStreamId = null; + if (context.S.session) context.S.session.active_stream_id = null; + return { ok: true, status: 200, json: async () => ({ cancelled: true }) }; + } + throw new Error(`unexpected fetch: ${url}`); + } + + const storage = new Map(); + const context = { + AbortController, + console, + Uint8Array, + Int16Array, + Float32Array, + ArrayBuffer, + DataView, + Promise, + Math, + JSON, + String, + Number, + Error, + parseInt, + parseFloat, + isNaN, + Set, + Map, + Array, + Object, + URL: FakeURL, + queueMicrotask, + Date: { now: () => clock.now }, + Blob: function Blob(parts, blobOptions) { + this.parts = parts; + this.type = (blobOptions || {}).type || ''; + this.size = 512; + }, + File: function File(parts, name, fileOptions) { + this.parts = parts; this.name = name; this.type = (fileOptions || {}).type || ''; + }, + FormData: function FormData() { + this.entries = []; this.append = (key, value) => this.entries.push([key, value]); + }, + Audio: function Audio() { + this.play = () => Promise.resolve(); + this.pause = () => {}; + this.onended = null; + this.onerror = null; + }, + WebSocket: FakeWebSocket, + AudioWorkletNode: FakeAudioWorkletNode, + MediaRecorder: FakeMediaRecorder, + AudioContext: FakeAudioContext, + fetch: fetchStub, + localStorage: { + getItem: (key) => (storage.has(key) ? storage.get(key) : null), + setItem: (key, value) => { storage.set(key, String(value)); }, + removeItem: (key) => { storage.delete(key); }, + }, + navigator: { + mediaDevices: { + getSupportedConstraints: () => ({}), + getUserMedia: async () => { + const track = { + stop() {}, + enabled: true, + getSettings: () => ({ echoCancellation: true }), + }; + micTracks.push(track); + return { getTracks: () => [track], getAudioTracks: () => [track] }; + }, + }, + }, + document: { + baseURI: 'https://chat.test/', + body: bodyElement, + getElementById: (id) => elements[id] || null, + querySelectorAll: () => assistantRows, + createElement: (tag) => makeElement(`created-${tag}`), + addEventListener() {}, + removeEventListener() {}, + }, + location: { protocol: 'https:', host: 'chat.test', href: 'https://chat.test/' }, + crypto: { getRandomValues(bytes) { for (let i = 0; i < bytes.length; i += 1) bytes[i] = i + 1; } }, + S: { session: { session_id: 'session-1' }, busy: false, activeStreamId: null }, + setTimeout: (fn, delay) => { + const id = timerId; timerId += 1; + timeouts.push({ id, fn, at: clock.now + (delay || 0) }); + return id; + }, + clearTimeout: (id) => { + const index = timeouts.findIndex((entry) => entry.id === id); + if (index >= 0) timeouts.splice(index, 1); + }, + setInterval: (fn, delay) => { + const id = timerId; timerId += 1; + intervals.set(id, { fn, delay: delay || 0 }); + return id; + }, + clearInterval: (id) => { intervals.delete(id); }, + }; + context.window = context; + context.window.location = context.location; + context.window.crypto = context.crypto; + context.showToast = (message) => { toasts.push(message); }; + context.send = () => { + streamCounter += 1; + sends.push(elements.msg.value); + context.S.busy = true; + context.S.activeStreamId = `stream-${streamCounter}`; + if (context.S.session) context.S.session.active_stream_id = context.S.activeStreamId; + }; + context.autoResize = () => {}; + context.stopTTS = () => {}; + context._stripForTTS = (text) => text; + + vm.createContext(context); + vm.runInContext(SOURCE, context, { filename: 'atlas-voice.js' }); + + function runDueTimeouts() { + const due = timeouts.filter((entry) => entry.at <= clock.now); + due.forEach((entry) => { + const index = timeouts.indexOf(entry); + if (index >= 0) timeouts.splice(index, 1); + entry.fn(); + }); + } + + function currentCaptureNode() { + for (let index = captureNodes.length - 1; index >= 0; index -= 1) { + if (captureNodes[index].connected) return captureNodes[index]; + } + return null; + } + + function deliverMicFrame() { + const node = currentCaptureNode(); + if (!node || !node.port.onmessage) return; + const samples = new Float32Array(TICK_SAMPLES); + const value = mic.amplitude / 32768; + for (let index = 0; index < samples.length; index += 1) { + samples[index] = index % 2 ? value : -value; + } + node.port.onmessage({ data: { type: 'pcm', samples: samples.buffer } }); + } + + async function tick(ms) { + const steps = Math.max(1, Math.round(ms / TICK_MS)); + for (let step = 0; step < steps; step += 1) { + clock.now += TICK_MS; + deliverMicFrame(); + Array.from(intervals.values()).forEach((entry) => entry.fn()); + runDueTimeouts(); + // eslint-disable-next-line no-await-in-loop + await flush(); + } + } + + async function speak(word, ms) { + const amplitude = Number(Object.keys(words).find((key) => words[key] === word)); + if (!amplitude) throw new Error(`unknown word: ${word}`); + mic.level = Math.min(0.9, amplitude / 32768 + 0.25); + mic.amplitude = amplitude; + await tick(ms); + } + + async function silence(ms) { + mic.level = 0; + mic.amplitude = 0; + await tick(ms); + } + + return { + context, + elements, + sends, + toasts, + transcribeUploads, + servers, + recorders, + clock, + tick, + speak, + silence, + flush, + runDueTimeouts, + setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; }, + clearAssistantRows() { assistantRows = []; }, + state() { return elements.voiceModeBar.dataset.voiceState || ''; }, + body() { return bodyElement; }, + trackStates() { return micTracks.map((track) => track.enabled); }, + overlay() { + return bodyElement.children.find( + (child) => String(child.className).indexOf('voice-conversation') >= 0, + ) || null; + }, + completeResponse() { + context.S.busy = false; + context.S.activeStreamId = null; + if (context.S.session) context.S.session.active_stream_id = null; + context.autoReadLastAssistant(); + }, + async start() { + await flush(); + elements.btnVoiceMode.click(); + await flush(); + await flush(); + }, + }; +} + +const scenarios = {}; + +// The reported live regression: a normal multi-word utterance with a natural +// inter-word pause long enough for the client to speculate (>=605ms at the +// default 1100ms endpoint) must still transcribe completely. +scenarios.multiword_utterance_with_interword_pause = async () => { + const harness = makeHarness(); + await harness.start(); + await harness.silence(300); + await harness.speak('alpha', 400); + await harness.silence(700); // client speculates at ~605ms + await harness.speak('bravo', 600); + await harness.silence(1400); // past the 1100ms endpoint + await harness.tick(400); // commit retries + final delivery + return { + sends: harness.sends, + state: harness.state(), + serverEvents: harness.servers.map((socket) => socket.server.events), + toasts: harness.toasts, + }; +}; + +// Continuous capture: the mic stays hot after an utterance is dispatched, and +// the next utterance (spoken while the first response is still in flight) +// barges in and is transcribed completely, including words after a pause. +scenarios.second_utterance_during_response_is_complete = async () => { + const harness = makeHarness(); + await harness.start(); + await harness.silence(300); + await harness.speak('alpha', 500); + await harness.silence(1400); + await harness.tick(400); // first send dispatched + const firstSends = harness.sends.slice(); + // The model is now answering (S.busy). Assistant text starts streaming. + harness.setAssistantReply('Partial answer already visible.'); + await harness.tick(200); + // User talks over the response with a two-word interruption + pause. + await harness.speak('bravo', 500); + await harness.silence(700); + await harness.speak('charlie', 500); + await harness.silence(1400); + await harness.tick(400); + return { + firstSends, + sends: harness.sends, + state: harness.state(), + serverEvents: harness.servers.map((socket) => socket.server.events), + toasts: harness.toasts, + }; +}; + +// The mic never re-acquires getUserMedia across utterances and a second +// utterance right after a normally completed response is sent alone: a +// normal completion clears any stitch context. +scenarios.back_to_back_utterances_stay_hot = async () => { + const harness = makeHarness(); + await harness.start(); + await harness.silence(300); + await harness.speak('alpha', 1300); + await harness.silence(1500); + await harness.tick(400); // first send dispatched (past the endpoint) + const sendsAfterFirst = harness.sends.slice(); + harness.completeResponse(); + await harness.tick(400); + await harness.speak('bravo', 1300); + await harness.silence(1500); + await harness.tick(400); + return { + sendsAfterFirst, + sends: harness.sends, + micAcquisitions: harness.trackStates().length, + recorderCount: harness.recorders.length, + serverCount: harness.servers.length, + toasts: harness.toasts, + }; +}; + +// The exact live regression: "The … rest of the sentence" +// used to endpoint on the first inter-word gap (>=1100ms) and send the first +// word alone. A young utterance now holds its endpoint to 1800ms, so the +// pause stays inside one utterance and one complete message is sent. +scenarios.thinking_pause_after_first_word_does_not_split = async () => { + const harness = makeHarness(); + await harness.start(); + await harness.silence(300); + await harness.speak('alpha', 400); + await harness.silence(1300); // longer than the old 1100ms endpoint + await harness.speak('bravo', 700); + await harness.silence(2000); + await harness.tick(400); + return { + sends: harness.sends, + serverEvents: harness.servers.map((socket) => socket.server.events), + toasts: harness.toasts, + }; +}; + +// Conversation mode: activation builds the full-screen overlay, captions +// follow the turn, mute drives the real capture track, and both the exit +// control and Escape tear the overlay down completely. +scenarios.conversation_overlay_lifecycle = async () => { + const harness = makeHarness(); + await harness.start(); + const overlay = harness.overlay(); + if (!overlay) return { overlayPresent: false }; + const find = (root, needle) => root.children.find( + (child) => String(child.className).indexOf(needle) >= 0, + ); + const role = overlay.getAttribute('role'); + const ariaModal = overlay.getAttribute('aria-modal'); + await harness.silence(300); + const listeningState = overlay.dataset.voiceState; + await harness.speak('alpha', 1300); + await harness.silence(1500); + await harness.tick(400); + const captions = find(overlay, 'voice-conversation-captions'); + const userCaption = captions ? captions.children[0].textContent : ''; + const captionsLive = captions ? captions.getAttribute('aria-live') : ''; + harness.setAssistantReply('A visible answer.'); + await harness.tick(300); + const assistantCaption = captions ? captions.children[1].textContent : ''; + const thinkingState = overlay.dataset.voiceState; + const controls = find(overlay, 'voice-conversation-controls'); + const muteBtn = controls.children[0]; + const exitBtn = controls.children[1]; + muteBtn.click(); + const mutedPressed = muteBtn.getAttribute('aria-pressed'); + const mutedTracks = harness.trackStates(); + muteBtn.click(); + const unmutedTracks = harness.trackStates(); + exitBtn.click(); + await harness.flush(); + const removedOnExit = !harness.overlay(); + const inactiveAfterExit = !harness.context._voiceModeActive(); + // Re-activate and leave through Escape instead. + harness.elements.btnVoiceMode.click(); + await harness.flush(); + const overlayAgain = harness.overlay(); + const keydown = overlayAgain.listeners.find((entry) => entry.type === 'keydown'); + keydown.handler({ key: 'Escape', preventDefault() {} }); + await harness.flush(); + return { + overlayPresent: true, + role, + ariaModal, + listeningState, + thinkingState, + userCaption, + assistantCaption, + captionsLive, + mutedPressed, + mutedTracks, + unmutedTracks, + removedOnExit, + inactiveAfterExit, + removedOnEscape: !harness.overlay(), + sends: harness.sends, + }; +}; + +(async () => { + const output = {}; + for (const name of Object.keys(scenarios)) { + // eslint-disable-next-line no-await-in-loop + output[name] = await scenarios[name](); + } + process.stdout.write(JSON.stringify(output, null, 1)); +})().catch((error) => { + process.stderr.write(String((error && error.stack) || error)); + process.exit(1); +}); diff --git a/testing/tests/test_hermes_voice_capture_continuity.py b/testing/tests/test_hermes_voice_capture_continuity.py new file mode 100644 index 00000000..14dd28e8 --- /dev/null +++ b/testing/tests/test_hermes_voice_capture_continuity.py @@ -0,0 +1,158 @@ +"""Continuous-capture contracts: no first-word clipping, stitching, overlay. + +The scenarios drive the real ``atlas-voice.js`` against a stub streaming STT +WebSocket whose VAD, speculative EOS-freeze, resume and commit semantics +mirror ``hermes-jetson-stt-server.py``. Words are encoded as distinct PCM +amplitudes, so the transcript of a commit is exactly the words whose audio +survived endpointing — the live "only the first word was sent" regression is +directly observable in the composer text the probe records. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +VOICE_SCRIPT = ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js" +PROBE = ROOT / "testing" / "probes" / "hermes_voice_capture_probe.js" + + +@pytest.fixture(scope="module") +def probe_results() -> dict: + node = shutil.which("node") + if not node: + pytest.skip("node is required to drive the capture continuity contract") + completed = subprocess.run( + [node, str(PROBE), str(VOICE_SCRIPT)], + check=False, + capture_output=True, + text=True, + timeout=180, + ) + assert completed.returncode == 0, completed.stderr + return json.loads(completed.stdout) + + +def test_multiword_utterance_with_interword_pause_is_complete(probe_results): + scenario = probe_results["multiword_utterance_with_interword_pause"] + assert scenario["sends"] == ["alpha bravo"] + assert scenario["state"] == "thinking" + + +def test_thinking_pause_after_first_word_does_not_split(probe_results): + """A >1100ms pause right after the first word stays inside one utterance.""" + scenario = probe_results["thinking_pause_after_first_word_does_not_split"] + assert scenario["sends"] == ["alpha bravo"] + # The pause froze a first-word EOS snapshot on the server; resumed speech + # must have cleared it before the commit. + events = scenario["serverEvents"][0] + assert "resume-clear" in events or "recover-by-rms" in events + + +def test_barge_in_stitches_regardless_of_partial_assistant_output(probe_results): + scenario = probe_results["second_utterance_during_response_is_complete"] + assert scenario["firstSends"] == ["alpha"] + # Assistant text was already visible when the user talked over the + # response: the interrupted thought and the follow-up still form one + # stitched message. + assert scenario["sends"] == ["alpha", "alpha bravo charlie"] + + +def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results): + scenario = probe_results["back_to_back_utterances_stay_hot"] + assert scenario["sendsAfterFirst"] == ["alpha"] + assert scenario["sends"] == ["alpha", "bravo"] + # The whole session runs on one getUserMedia lease. + assert scenario["micAcquisitions"] == 1 + + +def test_conversation_overlay_lifecycle(probe_results): + scenario = probe_results["conversation_overlay_lifecycle"] + assert scenario["overlayPresent"] is True + assert scenario["role"] == "dialog" + assert scenario["ariaModal"] == "true" + assert scenario["captionsLive"] == "polite" + assert scenario["listeningState"] == "listening" + assert scenario["thinkingState"] in {"thinking", "speaking"} + assert scenario["userCaption"] == "alpha" + assert scenario["assistantCaption"] == "A visible answer." + # Mute drives the real capture track and is fully reversible. + assert scenario["mutedPressed"] == "true" + assert scenario["mutedTracks"] == [False] + assert scenario["unmutedTracks"] == [True] + # Both exit paths remove the overlay completely and end hands-free mode. + assert scenario["removedOnExit"] is True + assert scenario["inactiveAfterExit"] is True + assert scenario["removedOnEscape"] is True + + +def test_overlay_source_contract(): + source = VOICE_SCRIPT.read_text(encoding="utf-8") + css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text( + encoding="utf-8" + ) + + # Lazily created, fully removed, storage-free, single-capture overlay. + assert "function openConversationOverlay()" in source + assert "function removeConversationOverlay()" in source + assert source.count("navigator.mediaDevices.getUserMedia(") == 1 + overlay_region = source.split("Conversation mode overlay", 1)[1].split( + "function clearErrorTimer", 1 + )[0] + assert "localStorage" not in overlay_region + assert "track.enabled=!conversation.muted" in overlay_region + assert "'aria-label':'Voice conversation'" in overlay_region + assert "'aria-live':'polite'" in overlay_region + assert "event.key==='Escape'" in overlay_region + assert "event.key==='Tab'" in overlay_region + + # Orb styling: breathing, level-driven, playback-driven, reduced-motion. + assert ".voice-conversation" in css + assert "--conversation-level" in css + assert "voice-conversation-breathe" in css + assert ".voice-conversation.is-playing .voice-conversation-orb-halo" in css + reduced = css.split("@media (prefers-reduced-motion: reduce)", 1)[1] + assert ".voice-conversation *" in reduced + + +def test_endpointing_and_streaming_hardening_source_contract(): + source = VOICE_SCRIPT.read_text(encoding="utf-8") + + # Young utterances hold their endpoint past a thinking pause. + assert "const VAD_EARLY_SILENCE_MS=1800" in source + assert "const VAD_COMMITTED_SPEECH_MS=1200" in source + assert ( + "const endpointSilenceMs=speechMs=SERVER_EOS_SILENCE_MS" in source + assert "if(settled||committed||!speculative) return;" not in source.split( + "resume:function()", 1 + )[1].split("speculate:function()", 1)[0] + # No speculative Whisper pass on a one-word fragment. + assert "const SPECULATE_MIN_SPEECH_MS=700" in source + assert "speechMs>=SPECULATE_MIN_SPEECH_MS" in source + # Echo residue can no longer ratchet the onset threshold above speech. + assert "Math.min(rms,speechThreshold)*0.06" in source + + +def test_stitch_gate_ignores_partial_assistant_output(): + source = VOICE_SCRIPT.read_text(encoding="utf-8") + region = source.split("function recordPendingStitch()", 1)[1].split( + "async function sendTranscript", 1 + )[0] + assert "currentAssistantText()" not in region + assert "pendingStitch={text:sent.text,at:Date.now()}" in region + # Normal completion still clears the stitch context. + assert ( + "if(isFinal){\n // The completion callback marks a normally " + "completed response" in source + ) diff --git a/testing/tests/test_hermes_voice_instrument.py b/testing/tests/test_hermes_voice_instrument.py index b209f0f2..09dabb49 100644 --- a/testing/tests/test_hermes_voice_instrument.py +++ b/testing/tests/test_hermes_voice_instrument.py @@ -165,7 +165,10 @@ def test_visual_slice_preserves_private_voice_request_and_capture_contract(): assert "if(language) request.language=language" in script assert "speakResponse(generation)" in script assert "window._voiceModeImmediateSend" in script - assert "mute" not in script.lower() + # Conversation-mode mute must silence the retained capture tracks rather + # than opening a second microphone lease or a speaker-choice surface. + assert "track.enabled=!conversation.muted" in script + assert "hermes-tts-voice" not in script assert "const TTS_LANGUAGES=['en','ru','es']" in script