From 181a7517c2ea1176a7c3dfe4635fcb9a910955d6 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 17:32:13 -0300 Subject: [PATCH] hermes(voice): interim-ack tail, natural fillers, unified audio + output picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three conversation-mode fixes in one pass: - Interim-acknowledgement truncation: when an interim ack folds into the hidden worklog segment mid-speech, the retained unspoken tail is now flushed and spoken in full before the Thinking transition, and a distinct follow-up message is chunked from its own start and queued after the interim drains (no more 'stops after the first clause, rest resurfaces with the next message'). - Natural thinking fillers: brief per-language interjections (Umm/Hmm/ One sec; Mmm/A ver; Хм/Секунду) on genuine >1.9s thinking gaps only, non-repeating, answer-preempting, mute-aware. - One unified audio sink for every spoken output (reply, cues, fillers, WAV fallback) - fixes cues playing the loudspeaker while the reply used a different output - plus a tidy corner output-device selector (enumerateDevices + setSinkId, feature-detected, session-only) styled like the language selector. Also realigns two STT-server decode-param assertions to the dict form from the STT tuning commit. 284 voice tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- dockerfiles/hermes-webui-atlas-voice.css | 114 ++++++ dockerfiles/hermes-webui-atlas-voice.js | 344 ++++++++++++++++-- testing/probes/hermes_voice_capture_probe.js | 337 +++++++++++++++++ testing/tests/test_hermes_chat_quality.py | 5 +- .../test_hermes_voice_capture_continuity.py | 134 ++++++- 5 files changed, 893 insertions(+), 41 deletions(-) diff --git a/dockerfiles/hermes-webui-atlas-voice.css b/dockerfiles/hermes-webui-atlas-voice.css index 70787890..98d4aff7 100644 --- a/dockerfiles/hermes-webui-atlas-voice.css +++ b/dockerfiles/hermes-webui-atlas-voice.css @@ -688,6 +688,120 @@ color: rgb(var(--voice-accent)); } +/* Output-device selector: a twin of the language chooser, seated just left of + it in the same corner. Feature-detected and hidden entirely when the browser + cannot enumerate/route outputs, so it never appears as a dead control. */ +.voice-conversation-out { + position: absolute; + top: calc(16px + env(safe-area-inset-top, 0px)); + right: calc(64px + env(safe-area-inset-right, 0px)); + z-index: 2; +} + +.voice-conversation-out-btn { + display: flex; + align-items: center; + justify-content: center; + width: 40px; + height: 40px; + padding: 0; + color: rgba(226, 238, 250, 0.72); + background: rgba(255, 255, 255, 0.06); + border: 1px solid rgba(var(--voice-accent), 0.28); + border-radius: 50%; + cursor: pointer; + transition: color 160ms ease, background 160ms ease, border-color 160ms ease; +} + +.voice-conversation-out-btn:hover { + color: rgba(240, 248, 255, 0.95); + background: rgba(255, 255, 255, 0.12); +} + +.voice-conversation-out-btn:focus-visible { + outline: 2px solid rgb(var(--voice-accent)); + outline-offset: 2px; +} + +/* A non-default output device is marked with the same accent dot. */ +.voice-conversation-out-btn.is-forced { + color: rgb(var(--voice-accent)); + border-color: rgba(var(--voice-accent), 0.7); +} + +.voice-conversation-out-btn.is-forced::after { + content: ""; + position: absolute; + top: 2px; + right: 2px; + width: 8px; + height: 8px; + border-radius: 50%; + background: rgb(var(--voice-accent)); + box-shadow: 0 0 6px rgba(var(--voice-accent), 0.8); +} + +.voice-conversation-out-menu { + position: absolute; + top: 46px; + right: 0; + min-width: 180px; + max-width: 260px; + display: flex; + flex-direction: column; + gap: 2px; + padding: 6px; + background: rgba(16, 22, 36, 0.96); + border: 1px solid rgba(var(--voice-accent), 0.28); + border-radius: 14px; + box-shadow: 0 18px 44px rgba(0, 0, 0, 0.5); + backdrop-filter: blur(8px); +} + +.voice-conversation-out-menu[hidden] { + display: none; +} + +.voice-conversation-out-item { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + font: inherit; + font-size: 14px; + font-weight: 550; + text-align: left; + color: rgba(231, 241, 251, 0.9); + background: transparent; + border: 0; + border-radius: 9px; + padding: 9px 12px; + cursor: pointer; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.voice-conversation-out-item:hover { + background: rgba(255, 255, 255, 0.08); +} + +.voice-conversation-out-item:focus-visible { + outline: 2px solid rgb(var(--voice-accent)); + outline-offset: -2px; +} + +.voice-conversation-out-item[aria-checked="true"] { + color: #fff; + background: rgba(var(--voice-accent), 0.16); +} + +.voice-conversation-out-item[aria-checked="true"]::after { + content: "✓"; + font-size: 13px; + color: rgb(var(--voice-accent)); +} + .voice-conversation.is-muted .voice-conversation-orb-halo, .voice-conversation.is-muted .voice-conversation-orb-core { animation: none; diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index 99d2d715..27933e58 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -36,6 +36,18 @@ let visualInputLevel=0; let streamingCapability={tts:null,stt:null,preflight:null}; let voicePreflight=null; + // Unified spoken-output sink. Every spoken sound — the reply's streaming PCM, + // the thinking-cue PCM, and the WAV blob fallback — plays through this ONE + // shared AudioContext (and, for the blob element, the same chosen sinkId), so + // the reply and its fillers can never land on different hardware outputs + // (the earpiece-vs-loudspeaker split). The context is created lazily on first + // playback, persists across cues/segments/turns, and is closed only when the + // hands-free session ends. selectedOutputSinkId is the user-chosen audio + // output device (session-only; '' means the system default). + let sharedPlaybackContext=null; + let sharedPlaybackWorklet=null; + let selectedOutputSinkId=''; + let outputDevices=[]; // Continuous-capture state. The microphone stream and its AudioContext stay // hot for the whole hands-free session; each utterance is one "capture turn" // guarded by captureGeneration so a response-side barge (which bumps @@ -100,10 +112,15 @@ const TTS_SPEED_MAX=2; const THINKING_CUE_FIRST_MS=1900; const THINKING_CUE_INTERVAL_MS=6500; + // Natural spoken fillers first (Umm/Hmm/One sec and their es/ru equivalents), + // then the longer reassurances — one short filler per genuine >~1.9s thinking + // gap (THINKING_CUE_FIRST_MS), the answer always preempting via + // cancelThinkingCues, rotated deterministically per turn so they never repeat + // back-to-back, spoken through the SAME unified sink as the reply. const THINKING_CUE_POOLS={ - en:[{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}], - ru:[{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}], - es:[{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}], + en:[{id:'umm',text:'Umm.'},{id:'hmm',text:'Hmm.'},{id:'one_sec',text:'One sec.'},{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}], + ru:[{id:'hmm',text:'Хм.'},{id:'sec',text:'Секунду.'},{id:'minute',text:'Минутку.'},{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}], + es:[{id:'mmm',text:'Mmm.'},{id:'a_ver',text:'A ver.'},{id:'un_momento',text:'Un momento.'},{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}], }; const STATE_LABELS={ listening:'Listening', @@ -145,6 +162,7 @@ {code:'ru',label:'Русский',sublabel:'Russian'}, ]; const GLOBE_ICON_SVG=''; + const SPEAKER_ICON_SVG=''; const originalAutoRead=window.autoReadLastAssistant; const originalApplyPreference=window._applyVoiceModePref; @@ -350,12 +368,15 @@ // Escape peels one layer at a time: an open language menu closes first // (focus returns to its button), and only a second Escape exits the mode. if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;} + if(conversation.outMenu&&!conversation.outMenu.hidden){closeOutputMenu(true);return;} deactivate(true); return; } if(event.key==='Tab'){ - // Minimal focus trap across the overlay controls (language, mute, exit). - const stops=[conversation.langBtn,conversation.muteBtn,conversation.exitBtn].filter(Boolean); + // Minimal focus trap across the overlay controls (language, output, mute, + // exit); the output button only joins the trap once it is actually shown. + const outVisible=conversation.outWrap&&conversation.outWrap.style&&conversation.outWrap.style.display!=='none'; + const stops=[conversation.langBtn,outVisible?conversation.outBtn:null,conversation.muteBtn,conversation.exitBtn].filter(Boolean); const current=stops.indexOf(document.activeElement); const index=current<0?(event.shiftKey?0:stops.length-1):current; const next=stops[(index+(event.shiftKey?stops.length-1:1))%stops.length]; @@ -451,6 +472,118 @@ return {wrap:wrap,btn:btn,menu:menu,items:items}; } + // ── Output-device selector ──────────────────────────────────────────── + // A tidy corner control, styled like the language chooser, to pick which audio + // output device the voice plays through. Feature-detected (enumerateDevices + + // setSinkId); the whole control stays hidden when routing is unsupported or no + // output devices are enumerable. Session-only — it never touches storage. + function closeOutputMenu(focusButton){ + if(!conversation||!conversation.outMenu) return; + conversation.outMenu.hidden=true; + conversation.outMenu.setAttribute('hidden',''); + if(conversation.outBtn){ + conversation.outBtn.setAttribute('aria-expanded','false'); + if(focusButton&&conversation.outBtn.focus) conversation.outBtn.focus(); + } + } + + function openOutputMenu(){ + if(!conversation||!conversation.outMenu) return; + conversation.outMenu.hidden=false; + conversation.outMenu.removeAttribute('hidden'); + if(conversation.outBtn) conversation.outBtn.setAttribute('aria-expanded','true'); + const items=conversation.outItems||[]; + const active=items.filter(function(item){return item.getAttribute('aria-checked')==='true';})[0]||items[0]; + if(active&&active.focus) active.focus(); + } + + function toggleOutputMenu(){ + if(!conversation||!conversation.outMenu) return; + if(conversation.outMenu.hidden) openOutputMenu(); else closeOutputMenu(true); + } + + function outputDeviceLabel(device,index){ + const label=device&&device.label?String(device.label):''; + if(label) return label; + return 'Output '+(index+1); + } + + function reflectOutputSelection(){ + if(!conversation||!conversation.outItems) return; + conversation.outItems.forEach(function(item){ + const selected=(item.getAttribute('data-device')||'')===selectedOutputSinkId; + item.setAttribute('aria-checked',selected?'true':'false'); + }); + if(conversation.outBtn){ + const chosen=outputDevices.filter(function(device){return device.deviceId===selectedOutputSinkId;})[0]; + const label=selectedOutputSinkId&&chosen?outputDeviceLabel(chosen,0):'System default'; + conversation.outBtn.setAttribute('aria-label','Audio output: '+label); + conversation.outBtn.setAttribute('title','Output — '+label); + if(conversation.outBtn.classList){ + if(selectedOutputSinkId) conversation.outBtn.classList.add('is-forced'); + else conversation.outBtn.classList.remove('is-forced'); + } + } + } + + function selectOutputDevice(deviceId){ + selectedOutputSinkId=deviceId||''; + reflectOutputSelection(); + applyOutputSink(); + } + + async function refreshOutputDevices(){ + if(!conversation||!conversation.outMenu||!conversation.outWrap) return; + if(!outputRoutingSupported()){conversation.outWrap.style.display='none';return;} + let devices=[]; + try{devices=await navigator.mediaDevices.enumerateDevices();}catch(_){devices=[];} + // Real, routable outputs only: the empty-deviceId "default" device the + // browser reports is represented by our own "System default" entry. + outputDevices=(devices||[]).filter(function(device){return device&&device.kind==='audiooutput'&&device.deviceId;}); + if(!conversation||!conversation.outMenu) return; + // Rebuild the menu: a "System default" entry plus every routable output. + conversation.outMenu.children=[]; + const entries=[{deviceId:'',label:'System default'}].concat( + outputDevices.map(function(device,index){return {deviceId:device.deviceId,label:outputDeviceLabel(device,index)};}) + ); + conversation.outItems=entries.map(function(entry){ + const item=conversationNode('button','voice-conversation-out-item',{ + type:'button',role:'menuitemradio','data-device':entry.deviceId, + 'aria-checked':entry.deviceId===selectedOutputSinkId?'true':'false', + }); + item.textContent=entry.label; + item.addEventListener('click',function(){ + selectOutputDevice(entry.deviceId); + closeOutputMenu(true); + }); + conversation.outMenu.appendChild(item); + return item; + }); + // Only the default entry means nothing is really selectable — hide it. + conversation.outWrap.style.display=outputDevices.length?'':'none'; + reflectOutputSelection(); + } + + function buildOutputControl(){ + const wrap=conversationNode('div','voice-conversation-out'); + // Hidden until refreshOutputDevices confirms real, routable outputs exist. + wrap.style.display='none'; + const btn=conversationNode('button','voice-conversation-out-btn',{ + type:'button','aria-haspopup':'menu','aria-expanded':'false', + 'aria-label':'Audio output: System default',title:'Output — System default', + }); + btn.innerHTML=SPEAKER_ICON_SVG; + const menu=conversationNode('div','voice-conversation-out-menu',{role:'menu','aria-label':'Audio output device',hidden:''}); + menu.hidden=true; + btn.addEventListener('click',function(event){ + if(event&&event.stopPropagation) event.stopPropagation(); + toggleOutputMenu(); + }); + wrap.appendChild(btn); + wrap.appendChild(menu); + return {wrap:wrap,btn:btn,menu:menu}; + } + function openConversationOverlay(){ if(conversation||!conversationUsable()) return; try{ @@ -485,6 +618,8 @@ controls.appendChild(exitBtn); const lang=buildLanguageControl(); root.appendChild(lang.wrap); + const out=buildOutputControl(); + root.appendChild(out.wrap); root.appendChild(orb); root.appendChild(stateEl); root.appendChild(captions); @@ -494,14 +629,16 @@ root.addEventListener('keydown',conversationKeydown); // A tap anywhere outside the language menu closes it (never deactivates). root.addEventListener('click',function(event){ - if(!conversation||!conversation.langMenu||conversation.langMenu.hidden) return; + if(!conversation) return; const target=event&&event.target; - const inWrap=target&&typeof target.closest==='function'&&target.closest('.voice-conversation-lang'); - if(!inWrap) closeLanguageMenu(false); + const closest=target&&typeof target.closest==='function'?target.closest.bind(target):null; + if(conversation.langMenu&&!conversation.langMenu.hidden&&!(closest&&closest('.voice-conversation-lang'))) closeLanguageMenu(false); + if(conversation.outMenu&&!conversation.outMenu.hidden&&!(closest&&closest('.voice-conversation-out'))) closeOutputMenu(false); }); document.body.appendChild(root); - conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,langBtn:lang.btn,langMenu:lang.menu,langItems:lang.items,muted:false}; + conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,langBtn:lang.btn,langMenu:lang.menu,langItems:lang.items,outWrap:out.wrap,outBtn:out.btn,outMenu:out.menu,outItems:[],muted:false}; reflectLanguageSelection(); + refreshOutputDevices(); syncConversationOverlay(state); if(root.focus) root.focus(); }catch(_){ @@ -626,7 +763,9 @@ if(cue.controller){cue.controller.abort();cue.controller=null;} if(cue.audioWake){const wake=cue.audioWake;cue.audioWake=null;wake();} if(cue.node){try{cue.node.port.postMessage({type:'cancel'});cue.node.disconnect();}catch(_){ }cue.node=null;} - if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;} + // The cue shares the ONE playback context; drop the reference but never + // close the shared sink here. + if(cue.context){cue.context=null;} cue.gain=null; indicator.classList.remove('is-playing'); indicator.classList.remove('is-ducked'); @@ -722,6 +861,9 @@ cancelSpeechTurn(); releaseMicrophone(); stopPlayback(); + closeSharedPlayback(); + selectedOutputSinkId=''; + outputDevices=[]; removeConversationOverlay(); modeBtn.classList.remove('active'); setState('error',message); @@ -786,7 +928,9 @@ if(playbackSession.node){ try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ } } - if(playbackSession.context){try{playbackSession.context.close();}catch(_){ }} + // The context is the shared spoken-output sink; it is closed only when + // hands-free mode ends (closeSharedPlayback), never per playback session. + playbackSession.context=null; playbackSession=null; } indicator.classList.remove('is-playing'); @@ -817,6 +961,9 @@ cancelSpeechTurn(); releaseMicrophone(); stopPlayback(); + closeSharedPlayback(); + selectedOutputSinkId=''; + outputDevices=[]; modeBtn.classList.remove('active'); setState('idle'); if(showMessage) toast('Hands-free voice mode off'); @@ -2015,12 +2162,70 @@ // Export the deterministic splitter as a narrow diagnostic/test seam. window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;}; + function outputRoutingSupported(){ + // The corner output-device selector needs to both ENUMERATE outputs and + // ROUTE to one. Feature-detected so unsupported browsers hide it entirely. + const Context=window.AudioContext||window.webkitAudioContext; + const md=navigator.mediaDevices; + const canRoute=(Context&&Context.prototype&&typeof Context.prototype.setSinkId==='function')|| + (window.HTMLMediaElement&&window.HTMLMediaElement.prototype&&typeof window.HTMLMediaElement.prototype.setSinkId==='function'); + return !!(md&&typeof md.enumerateDevices==='function'&&canRoute); + } + + async function applyContextSink(context){ + if(!context||!selectedOutputSinkId||typeof context.setSinkId!=='function') return; + try{await context.setSinkId(selectedOutputSinkId);}catch(_){ } + } + + async function acquirePlaybackContext(){ + // One shared AudioContext for ALL spoken output. Its native rate is left to + // the browser; the worklet resamples each push from Piper's rate, so cues + // and answers (even at different sample rates) share the exact same sink. + const Context=window.AudioContext||window.webkitAudioContext; + if(!Context) return null; + if(sharedPlaybackContext&&sharedPlaybackContext.state!=='closed'){ + if(sharedPlaybackWorklet){try{await sharedPlaybackWorklet;}catch(_){ }} + return sharedPlaybackContext; + } + let context; + try{context=new Context({latencyHint:'interactive'});}catch(_){context=new Context();} + sharedPlaybackContext=context; + sharedPlaybackWorklet=context.audioWorklet.addModule(WORKLET_URL); + try{await sharedPlaybackWorklet;}catch(error){ + if(sharedPlaybackContext===context){sharedPlaybackContext=null;sharedPlaybackWorklet=null;} + try{context.close();}catch(_){ } + throw error; + } + await applyContextSink(context); + return context; + } + + function applyOutputSink(){ + // Route the shared context and any in-flight blob element to the chosen sink. + applyContextSink(sharedPlaybackContext); + if(currentAudio&&typeof currentAudio.setSinkId==='function'){ + try{currentAudio.setSinkId(selectedOutputSinkId||'');}catch(_){ } + } + } + + function closeSharedPlayback(){ + const context=sharedPlaybackContext; + sharedPlaybackContext=null; + sharedPlaybackWorklet=null; + if(context){try{context.close();}catch(_){ }} + } + function playBlob(blob,token){ return new Promise(function(resolve,reject){ if(!active||token!==generation){resolve();return;} const session=playbackSession; const url=URL.createObjectURL(blob); const audio=new Audio(url); + // Same chosen output as the PCM path: the WAV fallback element is routed to + // the selected sink so it can never split onto a different device. + if(selectedOutputSinkId&&typeof audio.setSinkId==='function'){ + try{audio.setSinkId(selectedOutputSinkId);}catch(_){ } + } currentAudio=audio; let settled=false; function cleanup(callback,value){ @@ -2092,7 +2297,8 @@ } function thinkingCueStillOwned(cue){ - return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking'); + // Mute-aware: a muted conversation (the user stepped away) never chatters. + return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking'&&!(conversation&&conversation.muted)); } function scheduleNextThinkingCue(cue,delay){ @@ -2123,10 +2329,10 @@ const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||'22050',10); if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000) throw new Error('Cached thinking cue metadata invalid'); if(!thinkingCueStillOwned(cue)) return; - const Context=window.AudioContext||window.webkitAudioContext; - try{cue.context=new Context({sampleRate:sampleRate,latencyHint:'interactive'});}catch(_){cue.context=new Context();} - await cue.context.audioWorklet.addModule(WORKLET_URL); - if(!thinkingCueStillOwned(cue)) return; + // The cue plays through the ONE shared spoken-output sink, exactly like the + // reply — never its own AudioContext (which could route to another device). + cue.context=await acquirePlaybackContext(); + if(!cue.context||!thinkingCueStillOwned(cue)) return; cue.node=new AudioWorkletNode(cue.context,'atlas-pcm-playback'); cue.gain=cue.context.createGain(); cue.node.connect(cue.gain); @@ -2163,7 +2369,7 @@ } cue.controller=null; if(cue.node){try{cue.node.disconnect();}catch(_){ }cue.node=null;} - if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;} + if(cue.context){cue.context=null;} cue.gain=null; indicator.classList.remove('is-playing'); if(thinkingCueStillOwned(cue)) scheduleNextThinkingCue(cue,THINKING_CUE_INTERVAL_MS); @@ -2237,19 +2443,14 @@ if(session.sampleRate!==asset.sampleRate) throw new Error('Streaming speech sample rate changed mid-turn'); return session; } - const Context=window.AudioContext||window.webkitAudioContext; - let context; - try{ - // Keep Piper at its native rate and let the browser/audio device own any - // final hardware conversion; the worklet interpolator is a fallback. - context=new Context({sampleRate:asset.sampleRate,latencyHint:'interactive'}); - }catch(_){ - context=new Context(); - } - await context.audioWorklet.addModule(WORKLET_URL); - if(!active||token!==generation){context.close();return;} + // One shared sink for every spoken sound (reply, cues, blob fallback); the + // worklet resamples Piper's rate to the context's native rate. The shared + // context is never closed per-session — only when hands-free mode ends. + const context=await acquirePlaybackContext(); + if(!context) return null; + if(!active||token!==generation) return null; const node=new AudioWorkletNode(context,'atlas-pcm-playback'); - if(!session||session.cancelled){context.close();return;} + if(!session||session.cancelled) return null; const gain=context.createGain(); session.context=context; session.node=node; @@ -2334,7 +2535,7 @@ if(!session||!session.node) return; await drainPcmPlayback(session,token); if(session.node){try{session.node.disconnect();}catch(_){ }session.node=null;} - if(session.context){try{session.context.close();}catch(_){ }session.context=null;} + if(session.context){session.context=null;} session.gain=null; session.drained=null; indicator.classList.remove('is-playing'); @@ -2468,6 +2669,7 @@ idleTimer:null, consumed:0, first:true, + voiceResolved:false, queue:[], waiters:[], final:false, @@ -2477,6 +2679,26 @@ return speechTurn; } + function flushRetainedTail(turn){ + // An interim message we had BEGUN speaking just stopped being readable — it + // folded into the hidden .assistant-segment-worklog-source when the model's + // tool call began, so collectAssistantResponse() now returns '' or a + // distinct segment. Its retained sourceText still holds an unspoken tail + // (turn.consumed < turn.sourceText.length): speak that tail IN FULL, as + // final-quality chunks from the text the client already holds — never by + // re-reading the now-hidden DOM — so the whole acknowledgement is heard + // before the turn transitions to Thinking or a new segment is chunked. + // Marking it fully consumed makes a transient empty read that later recovers + // the SAME message a no-op instead of a double-speak. + if(!turn||typeof turn.sourceText!=='string') return false; + const tail=turn.sourceText.slice(turn.consumed).trim(); + if(!tail){turn.consumed=turn.sourceText.length;return false;} + const flushed=adaptiveChunks(tail,true,false); + enqueueSpeech(turn,flushed.chunks.length?flushed.chunks:[tail]); + turn.consumed=turn.sourceText.length; + return true; + } + function pumpAssistantResponse(token,isFinal){ if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return; if(isFinal){ @@ -2503,6 +2725,14 @@ } const text=response.text; if(!text){ + // The message we had begun speaking just stopped being readable (an + // interim acknowledgement folded into the hidden worklog source when the + // tool call began). Speak its retained tail in full FIRST, so the + // acknowledgement is never truncated and the queue stays non-empty — the + // idle fallback can no longer drop to Thinking with an unspoken tail out. + // Only for a message we have actually BEGUN speaking (consumed>0): a not- + // yet-started message must not be flushed whole on a transient empty read. + if(speechTurn&&speechTurn.consumed>0) flushRetainedTail(speechTurn); if(isFinal){ // First-sentence-stop guard: once a reply has begun speaking, the // completion callback can fire a frame before the settle re-render @@ -2538,7 +2768,7 @@ // of the answer queue. cancelThinkingCues(); const turn=ensureSpeechTurn(token); - if(turn.first){ + if(!turn.voiceResolved){ // Reply-voice routing, finalized when the first chunk is cut: the // turn's STT-detected language wins, but script evidence in the reply // text corrects a wrong or missing detection (a Spanish reply must @@ -2551,14 +2781,33 @@ const resolved=forcedLanguage||strongReplyLanguage(text)||turn.sttLanguage||detectReplyLanguage(text); if(resolved) turn.language=resolved; if(!forcedLanguage&&!turn.sttLanguage&&resolved) sessionLanguage=resolved; + // One audio timeline, one voice: a new logical message after an interim + // fold keeps the language already committed to the queue (see the + // new-message reset below, which sets turn.first but never re-resolves). + turn.voiceResolved=true; } if(text.length0){ + const spokenPrefix=turn.sourceText.slice(0,turn.consumed); + if(turn.consumed>0&&spokenPrefix&&!text.startsWith(spokenPrefix)){ + // A DISTINCT new message replaced the one we were speaking: the interim + // acknowledgement folded to the hidden worklog source and the final + // answer rendered as its own segment (the already-spoken interim prefix + // is no longer a prefix of what we read). Speak the interim's retained + // tail in full, then fall through and process `text` as a BRAND-NEW + // message — its chunk offset resets against its OWN text and its audio + // queues AFTER the interim drains. Never advance the interim's consumed + // offset into the new message's text (that resurfaced the tail out of + // order). The voice is kept (turn.voiceResolved), not re-resolved. + flushRetainedTail(turn); + turn.sourceText=''; + turn.consumed=0; + turn.first=true; + }else if(turn.consumed>0){ + // Same message, unspoken tail revised by the renderer. Already-spoken + // text is immutable. On the completion callback, advance from the old + // spoken offset and close the queue even when the renderer rewrote an + // earlier span; replaying the revised prefix would be more disruptive + // than preserving the already-heard words. if(!isFinal) return; thinkingSession=null; thinkingTurnId=''; @@ -2571,8 +2820,9 @@ } finishSpeechQueue(turn); return; + }else{ + turn.sourceText=''; } - turn.sourceText=''; } turn.sourceText=text; const remaining=text.slice(turn.consumed).trimStart(); @@ -2633,6 +2883,8 @@ clearSttLanguage(); sessionLanguage=''; forcedLanguage=''; + selectedOutputSinkId=''; + outputDevices=[]; finalizeAttempts=0; modeBtn.classList.add('active'); toast('Hands-free private voice mode on'); @@ -2719,6 +2971,22 @@ adaptiveChunks:adaptiveChunks, sentenceEnd:sentenceEnd, cleanForSpeech:cleanForSpeech, + // Read-only, behaviour-neutral: lets a deterministic probe observe that the + // interim fold never leaves an unspoken tail outstanding while the state + // has fallen back to Thinking (the dropped-tail regression). + speechTurnSnapshot:function(){ + if(!speechTurn) return null; + return { + state:state, + consumed:speechTurn.consumed, + sourceLength:(speechTurn.sourceText||'').length, + queue:speechTurn.queue.slice(), + queueLength:speechTurn.queue.length, + speakingChunk:speechTurn.speakingChunk||'', + waiterLength:speechTurn.waiters.length, + final:!!speechTurn.final, + }; + }, }; } diff --git a/testing/probes/hermes_voice_capture_probe.js b/testing/probes/hermes_voice_capture_probe.js index 4a12e58e..479da579 100644 --- a/testing/probes/hermes_voice_capture_probe.js +++ b/testing/probes/hermes_voice_capture_probe.js @@ -110,6 +110,150 @@ function makeElement(id) { return element; } +// ── Faithful assistant-turn DOM ───────────────────────────────────────────── +// A minimal but real querySelectorAll / closest / getAttribute / hidden / +// recursive-textContent implementation, so the interim-acknowledgement FOLD can +// be reproduced exactly as the live renderer performs it: the interim segment is +// re-tagged .assistant-segment-worklog-source + hidden + aria-hidden, at which +// point the script's real collectAssistantResponse() returns '' for it (the same +// extraction the response probe locks) and the final answer renders as its own +// new .assistant-segment. Supports the compound selectors the extraction uses. +function parseDomSelector(selector) { + return String(selector).split(',').map((group) => { + const term = group.trim(); + const parts = []; + const re = /([.#]?[\w-]+)|\[([\w-]+)(?:([~|^$*]?=)"?([^"\]]*)"?)?\]/g; + let m; + while ((m = re.exec(term))) { + if (m[1]) { + if (m[1][0] === '.') parts.push({ kind: 'class', value: m[1].slice(1) }); + else parts.push({ kind: 'tag', value: m[1].toLowerCase() }); + } else if (m[2]) { + parts.push({ kind: 'attr', name: m[2], op: m[3] || null, value: m[4] }); + } + } + return parts; + }).filter((parts) => parts.length); +} + +class DomNode { + constructor(tag) { + this.tag = String(tag || 'div').toLowerCase(); + this.className = ''; + this.attributes = new Map(); + this.children = []; + this.parentNode = null; + this.hidden = false; + this._text = ''; + const self = this; + this.dataset = new Proxy({}, { + get(_t, key) { + if (typeof key !== 'string') return undefined; + const attr = 'data-' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()); + return self.attributes.has(attr) ? self.attributes.get(attr) : undefined; + }, + has(_t, key) { + const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()); + return self.attributes.has(attr); + }, + }); + } + get classList() { + const el = this; + return { + add(name) { const s = new Set(el.className.split(/\s+/).filter(Boolean)); s.add(name); el.className = [...s].join(' '); }, + remove(name) { el.className = el.className.split(/\s+/).filter((v) => v && v !== name).join(' '); }, + contains(name) { return el.className.split(/\s+/).indexOf(name) >= 0; }, + }; + } + setAttribute(name, value) { + this.attributes.set(name, String(value)); + if (name === 'class') this.className = String(value); + if (name === 'hidden') this.hidden = true; + } + getAttribute(name) { + if (name === 'class') return this.className || null; + return this.attributes.has(name) ? this.attributes.get(name) : null; + } + appendChild(child) { child.parentNode = this; this.children.push(child); return child; } + set textContent(value) { this._text = String(value); this.children = []; } + get textContent() { + if (this.children.length) return this.children.map((c) => c.textContent).join(''); + return this._text; + } + _matchesTerm(parts) { + return parts.every((p) => { + if (p.kind === 'class') return this.classList.contains(p.value); + if (p.kind === 'tag') return this.tag === p.value; + if (p.kind === 'attr') { + if (!this.attributes.has(p.name)) return false; + return p.op ? this.attributes.get(p.name) === p.value : true; + } + return false; + }); + } + matches(selector) { return parseDomSelector(selector).some((parts) => this._matchesTerm(parts)); } + _walk(out) { for (const c of this.children) { out.push(c); c._walk(out); } return out; } + querySelectorAll(selector) { + const groups = parseDomSelector(selector); + return this._walk([]).filter((node) => groups.some((parts) => node._matchesTerm(parts))); + } + querySelector(selector) { const all = this.querySelectorAll(selector); return all.length ? all[0] : null; } + closest(selector) { + const groups = parseDomSelector(selector); + let node = this; + while (node) { if (groups.some((parts) => node._matchesTerm(parts))) return node; node = node.parentNode; } + return null; + } +} + +function domEl(tag, className, attrs, text) { + const node = new DomNode(tag); + if (className) node.setAttribute('class', className); + if (attrs) Object.keys(attrs).forEach((k) => node.setAttribute(k, attrs[k])); + if (text !== undefined) node.textContent = text; + return node; +} + +// A rendered assistant turn: role header, worklog chip, and an assistant-turn +// blocks container the interim/final segments hang off of. +function buildAssistantTurn() { + const turn = domEl('div', 'msg-row assistant-turn', { 'data-role': 'assistant', 'data-session-id': 'session-1' }); + const role = domEl('div', 'msg-role assistant'); + role.appendChild(domEl('div', 'role-icon assistant', null, 'H')); + role.appendChild(domEl('span', 'msg-role-name', null, 'Hermes')); + turn.appendChild(role); + const blocks = domEl('div', 'assistant-turn-blocks'); + turn.appendChild(blocks); + turn.blocks = blocks; + return turn; +} + +// A live (still-streaming) answer segment: no data-raw-text yet, so the script +// reads its answer .msg-body — exactly the interim acknowledgement path. +function addLiveAnswerSegment(turn, text) { + const seg = domEl('div', 'assistant-segment', { 'data-live-assistant': '1' }); + seg.appendChild(domEl('div', 'msg-body', null, text)); + turn.blocks.appendChild(seg); + return seg; +} + +// A settled answer segment carrying the renderer-stamped clean answer text. +function addSettledAnswerSegment(turn, text) { + const seg = domEl('div', 'assistant-segment', { 'data-raw-text': text }); + seg.appendChild(domEl('div', 'msg-body', null, text)); + turn.blocks.appendChild(seg); + return seg; +} + +// Fold an interim segment into the hidden worklog source, exactly as the live +// renderer does when a tool call begins: the extraction now excludes it. +function foldSegmentIntoWorklog(seg) { + seg.setAttribute('class', 'assistant-segment assistant-segment-worklog-source'); + seg.setAttribute('aria-hidden', 'true'); + seg.hidden = true; +} + // Faithful port of StreamingTranscription's endpointing/speculation contract. class StubSttServer { constructor(words) { @@ -243,6 +387,7 @@ function makeHarness(options = {}) { const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' }; const micTracks = []; + const sinkCalls = []; const elements = {}; ['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg'] @@ -280,6 +425,7 @@ function makeHarness(options = {}) { createMediaStreamSource() { return { connect() {}, disconnect() {} }; } createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; } resume() { return Promise.resolve(); } + setSinkId(id) { sinkCalls.push({ kind: 'context', sink: id }); return Promise.resolve(); } close() { this.closed = true; return Promise.resolve(); } } @@ -466,6 +612,7 @@ function makeHarness(options = {}) { Audio: function Audio() { this.play = () => Promise.resolve(); this.pause = () => {}; + this.setSinkId = (id) => { sinkCalls.push({ kind: 'audio', sink: id }); return Promise.resolve(); }; this.onended = null; this.onerror = null; }, @@ -482,6 +629,7 @@ function makeHarness(options = {}) { navigator: { mediaDevices: { getSupportedConstraints: () => ({}), + ...(options.outputs ? { enumerateDevices: async () => options.outputs } : {}), getUserMedia: async () => { const track = { stop() {}, @@ -610,6 +758,28 @@ function makeHarness(options = {}) { setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; }, setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; }, clearAssistantRows() { assistantRows = []; }, + // ── Interim-fold reproduction ───────────────────────────────────────── + // Stream an interim acknowledgement as a live answer segment, then FOLD it + // into the hidden worklog source (as the renderer does at the tool call) and + // render the final answer as its own new segment. + setInterimAck(text) { + const turn = buildAssistantTurn(); + this._foldTurn = turn; + this._interimSeg = addLiveAnswerSegment(turn, text); + assistantRows = [turn]; + }, + foldInterimAck() { if (this._interimSeg) foldSegmentIntoWorklog(this._interimSeg); }, + setFinalAnswer(text) { + if (!this._foldTurn) return; + addSettledAnswerSegment(this._foldTurn, text); + }, + speechTurnSnapshot() { + const internals = context.window.__atlasVoiceInternals; + return internals && typeof internals.speechTurnSnapshot === 'function' + ? internals.speechTurnSnapshot() : null; + }, + ttsTexts() { return ttsCalls.map((request) => request.text); }, + sinkCalls() { return sinkCalls.slice(); }, state() { return elements.voiceModeBar.dataset.voiceState || ''; }, body() { return bodyElement; }, trackStates() { return micTracks.map((track) => track.enabled); }, @@ -935,6 +1105,173 @@ scenarios.language_override_forces_stt_and_voice = async () => { }; }; +// THE FOLD REGRESSION. A quick spoken acknowledgement streams as an interim +// answer segment while a tool call is prepared. The pump chunks the first two +// sentences and RETAINS the still-streaming third as an unspoken tail; then the +// renderer folds the interim into the hidden .assistant-segment-worklog-source +// (extraction → '') and the final answer renders as its own new segment. +// +// Before the fix: the empty-read branch returned without flushing the retained +// tail (the acknowledgement's last words were dropped) and the speech queue +// drained with the state fallen back to Thinking while a tail was still +// outstanding; the final answer then resurfaced from the interim's stale offset +// (a garbled mid-string slice). After the fix: the whole acknowledgement is +// spoken, in order, BEFORE the distinct final answer, and the state never falls +// back to Thinking with an unspoken interim tail outstanding. +scenarios.interim_ack_fold_flushes_tail_before_final = async () => { + const harness = makeHarness(); + // The stub Audio never fires onended, so playback of the first chunk parks and + // the queue does not drain (the same reason barge_cut_marker only inspects the + // first synthesized chunk). The complete ordered set of chunks that WILL be + // spoken is therefore the chunks already handed to synthesis (ttsTexts, in + // order) followed by the chunks still queued behind the parked one, assembled + // as spokenOrder below. Every assertion is a QUEUE/STATE fact, independent of + // the playback stub. + const violations = []; + const record = () => { + const snap = harness.speechTurnSnapshot(); + if (snap && snap.state === 'thinking' && !snap.final && snap.consumed < snap.sourceLength) { + violations.push({ consumed: snap.consumed, sourceLength: snap.sourceLength }); + } + }; + const settle = async (steps) => { for (let i = 0; i < steps; i += 1) { await harness.tick(100); record(); } }; + + await harness.start(); + await harness.silence(300); + await harness.speak('alpha', 1300); + await harness.silence(1500); + await harness.tick(400); // dispatched; thinking; response observer running + + // Interim acknowledgement: two complete sentences plus a third still mid-word, + // so the pump enqueues S1+S2 and retains "One moment while I che" unspoken. + const interim = 'Sure thing. Let me look that up. One moment while I che'; + harness.setInterimAck(interim); + await settle(4); + + // The tool call begins: fold the interim into the hidden worklog source. + harness.foldInterimAck(); + await settle(6); + const afterFold = harness.speechTurnSnapshot(); + const queuedAfterFold = (afterFold && afterFold.queue) || []; + + // The final answer renders as its own new, distinct segment. + const finalAnswer = 'The weather today is sunny and warm. Enjoy your afternoon out there.'; + harness.setFinalAnswer(finalAnswer); + await settle(6); + harness.completeResponse(); + await settle(10); + + const finalSnap = harness.speechTurnSnapshot(); + const residualQueue = (finalSnap && finalSnap.queue) || []; + // The full ordered list of chunks that reach the speech queue for this turn. + const spokenOrder = harness.ttsTexts().concat(residualQueue); + const joined = spokenOrder.join(''); + const lastInterimIdx = (() => { + let idx = -1; + spokenOrder.forEach((t, i) => { if (/Sure thing|look that up|One moment|while I che/.test(t)) idx = i; }); + return idx; + })(); + const firstFinalIdx = spokenOrder.findIndex((t) => /weather|sunny|afternoon/.test(t)); + + return { + spokenOrder, + // After the fold, the interim's retained tail is the next thing queued, and + // it is queued BEFORE any final-answer chunk exists. + queuedAfterFold, + tailQueuedAfterFold: queuedAfterFold.some((t) => /while I che/.test(t)), + noFinalBeforeFold: !queuedAfterFold.some((t) => /weather|sunny|afternoon/.test(t)), + // (i) every interim sentence reaches the speech queue: none dropped. + interimS1Reached: /Sure thing/.test(joined), + interimS2Reached: /look that up/.test(joined), + interimTailReached: /while I che/.test(joined), + // (iii) the distinct final answer is fully queued too. + finalS1Reached: /weather today is sunny/.test(joined), + finalS2Reached: /Enjoy your afternoon/.test(joined), + // The final answer is queued from its OWN start, never resurfaced from the + // interim's stale offset ("nny and warm" is the buggy mid-string slice). + finalNotGarbled: !spokenOrder.some((t) => /^nny and warm/.test(t)), + // (ii) the interim finishes before the final message's chunks are enqueued. + interimBeforeFinal: lastInterimIdx >= 0 && firstFinalIdx >= 0 && lastInterimIdx < firstFinalIdx, + // (iv) the state never fell back to Thinking with an unspoken interim tail. + thinkingWithUnspokenTail: violations.length, + state: harness.state(), + }; +}; + +// UNIFIED OUTPUT SINK + device selector. When the browser can enumerate outputs +// and setSinkId, a tidy corner control lists them; choosing one routes ALL spoken +// output (the reply's blob element here, and — via the shared AudioContext — the +// PCM reply and thinking cues) to that single device. When the APIs are missing, +// the control hides entirely (a dead control never appears). +scenarios.output_device_selector_routes_spoken_output = async () => { + const harness = makeHarness({ + outputs: [ + { deviceId: '', kind: 'audiooutput', label: 'System default' }, + { deviceId: 'spk-1', kind: 'audiooutput', label: 'Speaker One' }, + { deviceId: 'spk-2', kind: 'audiooutput', label: 'Headphones Two' }, + { deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' }, + ], + }); + await harness.start(); + await harness.flush(); + await harness.flush(); + const overlay = harness.overlay(); + if (!overlay) return { overlayPresent: false }; + const outWrap = overlay.children.find( + (child) => String(child.className).indexOf('voice-conversation-out') >= 0, + ); + const outBtn = outWrap ? outWrap.children.find( + (child) => String(child.className).indexOf('voice-conversation-out-btn') >= 0, + ) : null; + const outMenu = outWrap ? outWrap.children.find( + (child) => String(child.className).indexOf('voice-conversation-out-menu') >= 0, + ) : null; + const shownAfterRefresh = outWrap ? outWrap.style.display !== 'none' : false; + const items = outMenu ? outMenu.children : []; + const itemLabels = items.map((item) => item.textContent); + const headphones = items.find((item) => item.getAttribute('data-device') === 'spk-2'); + if (headphones) headphones.click(); + const forcedChecked = headphones ? headphones.getAttribute('aria-checked') : null; + const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false; + // A reply now plays through the blob fallback element, which must be routed to + // the chosen sink. + await harness.silence(300); + await harness.speak('alpha', 1300); + await harness.silence(1500); + await harness.tick(400); + harness.setAssistantReply('A short spoken reply.'); + harness.completeResponse(); + await harness.tick(400); + const sinks = harness.sinkCalls(); + return { + overlayPresent: true, + supported: true, + shownAfterRefresh, + itemLabels, + forcedChecked, + btnForced, + audioRoutedTo: sinks.filter((c) => c.kind === 'audio').map((c) => c.sink), + }; +}; + +// When enumerateDevices/setSinkId are unavailable the selector is present in the +// DOM but hidden — never a dead control. +scenarios.output_selector_hidden_when_unsupported = async () => { + const harness = makeHarness(); + await harness.start(); + await harness.flush(); + const overlay = harness.overlay(); + if (!overlay) return { overlayPresent: false }; + const outWrap = overlay.children.find( + (child) => String(child.className).indexOf('voice-conversation-out') >= 0, + ); + return { + overlayPresent: true, + controlInDom: !!outWrap, + hidden: outWrap ? outWrap.style.display === 'none' : null, + }; +}; + (async () => { const output = {}; for (const name of Object.keys(scenarios)) { diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index aab72c62..7c952eac 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -361,11 +361,12 @@ def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script assert "recorder.start();" in voice_script assert "recorder.start(250)" not in voice_script + assert "function flushRetainedTail(turn)" in voice_script # folded interim tail is spoken, never dropped stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() assert "def _repetitive_token" in stt_server - assert "compression_ratio_threshold=2.0" in stt_server - assert "no_speech_threshold=0.5" in stt_server + assert '"compression_ratio_threshold": 2.0' in stt_server + assert '"no_speech_threshold": 0.5' in stt_server def test_worker_webui_rollouts_do_not_repeat_completed_state_migration(): diff --git a/testing/tests/test_hermes_voice_capture_continuity.py b/testing/tests/test_hermes_voice_capture_continuity.py index e44aa66d..3ad6d8a8 100644 --- a/testing/tests/test_hermes_voice_capture_continuity.py +++ b/testing/tests/test_hermes_voice_capture_continuity.py @@ -77,6 +77,138 @@ def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results): assert scenario["micAcquisitions"] == 1 +def test_interim_ack_fold_flushes_retained_tail_before_final(probe_results): + """The reported TTS-truncation bug: a quick spoken acknowledgement streams as + an interim segment, the pump chunks its first sentences and RETAINS the still- + streaming tail; then a tool call folds the interim into the hidden worklog + source and the final answer renders as its own segment. + + Before the fix the retained tail was dropped (the empty-read branch returned + without flushing it) and the queue drained with the state fallen back to + Thinking while a tail was outstanding, and the final answer resurfaced from + the interim's stale offset. After the fix the WHOLE acknowledgement reaches + the speech queue, in order, before the distinct final answer, and the state + never falls back to Thinking with an unspoken interim tail outstanding. + """ + scenario = probe_results["interim_ack_fold_flushes_tail_before_final"] + # (i) every interim sentence reaches the queue — the retained tail is flushed + # the moment the segment folds, not dropped. + assert scenario["interimS1Reached"] is True + assert scenario["interimS2Reached"] is True + assert scenario["interimTailReached"] is True, "the folded interim tail was dropped" + assert scenario["tailQueuedAfterFold"] is True + assert scenario["noFinalBeforeFold"] is True + # (ii) the interim finishes before the final message's chunks are enqueued. + assert scenario["interimBeforeFinal"] is True + # (iii) the distinct final answer is also fully queued, from its own start — + # never resurfaced from the interim's stale offset. + assert scenario["finalS1Reached"] is True + assert scenario["finalS2Reached"] is True + assert scenario["finalNotGarbled"] is True + assert scenario["spokenOrder"] == [ + "Sure thing.", + "Let me look that up.", + "One moment while I che", + "The weather today is sunny and warm.", + "Enjoy your afternoon out there.", + ] + # (iv) the state never flipped to Thinking with an unspoken interim tail out. + assert scenario["thinkingWithUnspokenTail"] == 0 + + +def test_interim_fold_tail_flush_source_contract(): + source = VOICE_SCRIPT.read_text(encoding="utf-8") + + # The retained tail is flushed as final-quality chunks from the text the + # client already holds — never by re-reading the folded/hidden DOM. + assert "function flushRetainedTail(turn)" in source + assert "const tail=turn.sourceText.slice(turn.consumed).trim();" in source + assert "const flushed=adaptiveChunks(tail,true,false);" in source + # The empty-read branch flushes the tail before anything else. + assert "if(speechTurn&&speechTurn.consumed>0) flushRetainedTail(speechTurn);" in source + # A distinct new message after the fold resets the offset against its OWN + # text (never advancing the interim's consumed offset into it). + assert "const spokenPrefix=turn.sourceText.slice(0,turn.consumed);" in source + assert "if(turn.consumed>0&&spokenPrefix&&!text.startsWith(spokenPrefix)){" in source + # One audio timeline, one voice: the new message keeps the resolved voice. + assert "if(!turn.voiceResolved){" in source + assert "turn.voiceResolved=true;" in source + + +def test_output_device_selector_routes_all_spoken_output(probe_results): + """Unified sink + selector: when the browser can enumerate outputs and + setSinkId, the corner control lists the routable outputs and choosing one + routes the spoken output (the blob element here) to that device.""" + scenario = probe_results["output_device_selector_routes_spoken_output"] + assert scenario["overlayPresent"] is True + assert scenario["shownAfterRefresh"] is True + assert scenario["itemLabels"] == ["System default", "Speaker One", "Headphones Two"] + # Selection is reflected accessibly and marked as a non-default override. + assert scenario["forcedChecked"] == "true" + assert scenario["btnForced"] is True + # The spoken reply is routed to the chosen sink. + assert scenario["audioRoutedTo"] == ["spk-2"] + + +def test_output_selector_hidden_when_unsupported(probe_results): + """No enumerateDevices/setSinkId: the control is in the DOM but hidden — it + never appears as a dead control.""" + scenario = probe_results["output_selector_hidden_when_unsupported"] + assert scenario["overlayPresent"] is True + assert scenario["controlInDom"] is True + assert scenario["hidden"] is True + + +def test_natural_fillers_unified_sink_and_output_selector_source_contract(): + source = VOICE_SCRIPT.read_text(encoding="utf-8") + css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(encoding="utf-8") + + # (2) Natural spoken fillers per language, with the longer reassurances kept. + assert "{id:'umm',text:'Umm.'}" in source + assert "{id:'hmm',text:'Hmm.'}" in source + assert "{id:'one_sec',text:'One sec.'}" in source + assert "{id:'mmm',text:'Mmm.'}" in source + assert "{id:'a_ver',text:'A ver.'}" in source + assert "{id:'hmm',text:'Хм.'}" in source + assert "{id:'sec',text:'Секунду.'}" in source + # Answer preempts a filler; a genuine >~1.9s gap gates the first one. + assert "const THINKING_CUE_FIRST_MS=1900" in source + assert "cancelThinkingCues();\n const turn=ensureSpeechTurn(token)" in source + # Mute-aware: a muted conversation never chatters. + assert "state==='thinking'&&!(conversation&&conversation.muted)" in source + + # (3) ONE shared sink for every spoken sound. The only AudioContext built for + # OUTPUT is inside acquirePlaybackContext; cues and PCM replies both use it. + assert "async function acquirePlaybackContext()" in source + assert "function closeSharedPlayback()" in source + assert "cue.context=await acquirePlaybackContext();" in source + assert "const context=await acquirePlaybackContext();" in source + # The shared context is never closed per playback session/cue. + assert "try{cue.context=new Context(" not in source + assert "context=new Context({sampleRate:asset.sampleRate" not in source + # The blob fallback element is routed to the same chosen sink. + assert "audio.setSinkId(selectedOutputSinkId)" in source + + # The output-device selector: feature-detected, session-only, styled. + assert "function outputRoutingSupported()" in source + assert "navigator.mediaDevices.enumerateDevices" in source + assert "function buildOutputControl()" in source + assert "function selectOutputDevice(deviceId)" in source + assert "role:'menuitemradio','data-device':entry.deviceId" in source + # Session-only: the chosen output never touches storage. + out_region = source.split("function buildOutputControl()", 1)[1].split( + "function openConversationOverlay", 1 + )[0] + assert "localStorage" not in out_region + for token in ( + ".voice-conversation-out", + ".voice-conversation-out-btn", + ".voice-conversation-out-menu", + ".voice-conversation-out-item", + ): + assert token in css + + def test_conversation_overlay_lifecycle(probe_results): scenario = probe_results["conversation_overlay_lifecycle"] assert scenario["overlayPresent"] is True @@ -308,7 +440,7 @@ def test_conversation_language_selector_source_and_style_contract(): assert "localStorage" not in source.split("CONVERSATION_LANGUAGES", 1)[1].split("function openConversationOverlay", 1)[0] # Escape peels the menu before exiting; the language button joins the trap. assert "if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}" in source - assert "[conversation.langBtn,conversation.muteBtn,conversation.exitBtn]" in source + assert "[conversation.langBtn,outVisible?conversation.outBtn:null,conversation.muteBtn,conversation.exitBtn]" in source for token in (".voice-conversation-lang-btn", ".voice-conversation-lang-menu", "menuitemradio"): pass # menu roles live in JS; assert CSS hooks below for token in (".voice-conversation-lang", ".voice-conversation-lang-btn", ".voice-conversation-lang-menu", ".voice-conversation-lang-item"):