hermes(voice): interim-ack tail, natural fillers, unified audio + output picker
Some checks failed
Tests / Declarative: Post Actions failed: 49, skipped: 72, passed: 3724
Some checks failed
Tests / Declarative: Post Actions failed: 49, skipped: 72, passed: 3724
Three conversation-mode fixes in one pass: - Interim-acknowledgement truncation: when an interim ack folds into the hidden worklog segment mid-speech, the retained unspoken tail is now flushed and spoken in full before the Thinking transition, and a distinct follow-up message is chunked from its own start and queued after the interim drains (no more 'stops after the first clause, rest resurfaces with the next message'). - Natural thinking fillers: brief per-language interjections (Umm/Hmm/ One sec; Mmm/A ver; Хм/Секунду) on genuine >1.9s thinking gaps only, non-repeating, answer-preempting, mute-aware. - One unified audio sink for every spoken output (reply, cues, fillers, WAV fallback) - fixes cues playing the loudspeaker while the reply used a different output - plus a tidy corner output-device selector (enumerateDevices + setSinkId, feature-detected, session-only) styled like the language selector. Also realigns two STT-server decode-param assertions to the dict form from the STT tuning commit. 284 voice tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
This commit is contained in:
parent
64b7bc55cf
commit
181a7517c2
@ -688,6 +688,120 @@
|
|||||||
color: rgb(var(--voice-accent));
|
color: rgb(var(--voice-accent));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Output-device selector: a twin of the language chooser, seated just left of
|
||||||
|
it in the same corner. Feature-detected and hidden entirely when the browser
|
||||||
|
cannot enumerate/route outputs, so it never appears as a dead control. */
|
||||||
|
.voice-conversation-out {
|
||||||
|
position: absolute;
|
||||||
|
top: calc(16px + env(safe-area-inset-top, 0px));
|
||||||
|
right: calc(64px + env(safe-area-inset-right, 0px));
|
||||||
|
z-index: 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-btn {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
width: 40px;
|
||||||
|
height: 40px;
|
||||||
|
padding: 0;
|
||||||
|
color: rgba(226, 238, 250, 0.72);
|
||||||
|
background: rgba(255, 255, 255, 0.06);
|
||||||
|
border: 1px solid rgba(var(--voice-accent), 0.28);
|
||||||
|
border-radius: 50%;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: color 160ms ease, background 160ms ease, border-color 160ms ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-btn:hover {
|
||||||
|
color: rgba(240, 248, 255, 0.95);
|
||||||
|
background: rgba(255, 255, 255, 0.12);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-btn:focus-visible {
|
||||||
|
outline: 2px solid rgb(var(--voice-accent));
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A non-default output device is marked with the same accent dot. */
|
||||||
|
.voice-conversation-out-btn.is-forced {
|
||||||
|
color: rgb(var(--voice-accent));
|
||||||
|
border-color: rgba(var(--voice-accent), 0.7);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-btn.is-forced::after {
|
||||||
|
content: "";
|
||||||
|
position: absolute;
|
||||||
|
top: 2px;
|
||||||
|
right: 2px;
|
||||||
|
width: 8px;
|
||||||
|
height: 8px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: rgb(var(--voice-accent));
|
||||||
|
box-shadow: 0 0 6px rgba(var(--voice-accent), 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-menu {
|
||||||
|
position: absolute;
|
||||||
|
top: 46px;
|
||||||
|
right: 0;
|
||||||
|
min-width: 180px;
|
||||||
|
max-width: 260px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 2px;
|
||||||
|
padding: 6px;
|
||||||
|
background: rgba(16, 22, 36, 0.96);
|
||||||
|
border: 1px solid rgba(var(--voice-accent), 0.28);
|
||||||
|
border-radius: 14px;
|
||||||
|
box-shadow: 0 18px 44px rgba(0, 0, 0, 0.5);
|
||||||
|
backdrop-filter: blur(8px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-menu[hidden] {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-item {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 10px;
|
||||||
|
font: inherit;
|
||||||
|
font-size: 14px;
|
||||||
|
font-weight: 550;
|
||||||
|
text-align: left;
|
||||||
|
color: rgba(231, 241, 251, 0.9);
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
border-radius: 9px;
|
||||||
|
padding: 9px 12px;
|
||||||
|
cursor: pointer;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-item:hover {
|
||||||
|
background: rgba(255, 255, 255, 0.08);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-item:focus-visible {
|
||||||
|
outline: 2px solid rgb(var(--voice-accent));
|
||||||
|
outline-offset: -2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-item[aria-checked="true"] {
|
||||||
|
color: #fff;
|
||||||
|
background: rgba(var(--voice-accent), 0.16);
|
||||||
|
}
|
||||||
|
|
||||||
|
.voice-conversation-out-item[aria-checked="true"]::after {
|
||||||
|
content: "✓";
|
||||||
|
font-size: 13px;
|
||||||
|
color: rgb(var(--voice-accent));
|
||||||
|
}
|
||||||
|
|
||||||
.voice-conversation.is-muted .voice-conversation-orb-halo,
|
.voice-conversation.is-muted .voice-conversation-orb-halo,
|
||||||
.voice-conversation.is-muted .voice-conversation-orb-core {
|
.voice-conversation.is-muted .voice-conversation-orb-core {
|
||||||
animation: none;
|
animation: none;
|
||||||
|
|||||||
@ -36,6 +36,18 @@
|
|||||||
let visualInputLevel=0;
|
let visualInputLevel=0;
|
||||||
let streamingCapability={tts:null,stt:null,preflight:null};
|
let streamingCapability={tts:null,stt:null,preflight:null};
|
||||||
let voicePreflight=null;
|
let voicePreflight=null;
|
||||||
|
// Unified spoken-output sink. Every spoken sound — the reply's streaming PCM,
|
||||||
|
// the thinking-cue PCM, and the WAV blob fallback — plays through this ONE
|
||||||
|
// shared AudioContext (and, for the blob element, the same chosen sinkId), so
|
||||||
|
// the reply and its fillers can never land on different hardware outputs
|
||||||
|
// (the earpiece-vs-loudspeaker split). The context is created lazily on first
|
||||||
|
// playback, persists across cues/segments/turns, and is closed only when the
|
||||||
|
// hands-free session ends. selectedOutputSinkId is the user-chosen audio
|
||||||
|
// output device (session-only; '' means the system default).
|
||||||
|
let sharedPlaybackContext=null;
|
||||||
|
let sharedPlaybackWorklet=null;
|
||||||
|
let selectedOutputSinkId='';
|
||||||
|
let outputDevices=[];
|
||||||
// Continuous-capture state. The microphone stream and its AudioContext stay
|
// Continuous-capture state. The microphone stream and its AudioContext stay
|
||||||
// hot for the whole hands-free session; each utterance is one "capture turn"
|
// hot for the whole hands-free session; each utterance is one "capture turn"
|
||||||
// guarded by captureGeneration so a response-side barge (which bumps
|
// guarded by captureGeneration so a response-side barge (which bumps
|
||||||
@ -100,10 +112,15 @@
|
|||||||
const TTS_SPEED_MAX=2;
|
const TTS_SPEED_MAX=2;
|
||||||
const THINKING_CUE_FIRST_MS=1900;
|
const THINKING_CUE_FIRST_MS=1900;
|
||||||
const THINKING_CUE_INTERVAL_MS=6500;
|
const THINKING_CUE_INTERVAL_MS=6500;
|
||||||
|
// Natural spoken fillers first (Umm/Hmm/One sec and their es/ru equivalents),
|
||||||
|
// then the longer reassurances — one short filler per genuine >~1.9s thinking
|
||||||
|
// gap (THINKING_CUE_FIRST_MS), the answer always preempting via
|
||||||
|
// cancelThinkingCues, rotated deterministically per turn so they never repeat
|
||||||
|
// back-to-back, spoken through the SAME unified sink as the reply.
|
||||||
const THINKING_CUE_POOLS={
|
const THINKING_CUE_POOLS={
|
||||||
en:[{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}],
|
en:[{id:'umm',text:'Umm.'},{id:'hmm',text:'Hmm.'},{id:'one_sec',text:'One sec.'},{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}],
|
||||||
ru:[{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}],
|
ru:[{id:'hmm',text:'Хм.'},{id:'sec',text:'Секунду.'},{id:'minute',text:'Минутку.'},{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}],
|
||||||
es:[{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}],
|
es:[{id:'mmm',text:'Mmm.'},{id:'a_ver',text:'A ver.'},{id:'un_momento',text:'Un momento.'},{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}],
|
||||||
};
|
};
|
||||||
const STATE_LABELS={
|
const STATE_LABELS={
|
||||||
listening:'Listening',
|
listening:'Listening',
|
||||||
@ -145,6 +162,7 @@
|
|||||||
{code:'ru',label:'Русский',sublabel:'Russian'},
|
{code:'ru',label:'Русский',sublabel:'Russian'},
|
||||||
];
|
];
|
||||||
const GLOBE_ICON_SVG='<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.5" aria-hidden="true" focusable="false"><circle cx="10" cy="10" r="7.25"/><path d="M2.9 10h14.2M10 2.75c1.9 2 2.9 4.6 2.9 7.25S11.9 15.25 10 17.25C8.1 15.25 7.1 12.65 7.1 10S8.1 4.75 10 2.75z"/></svg>';
|
const GLOBE_ICON_SVG='<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.5" aria-hidden="true" focusable="false"><circle cx="10" cy="10" r="7.25"/><path d="M2.9 10h14.2M10 2.75c1.9 2 2.9 4.6 2.9 7.25S11.9 15.25 10 17.25C8.1 15.25 7.1 12.65 7.1 10S8.1 4.75 10 2.75z"/></svg>';
|
||||||
|
const SPEAKER_ICON_SVG='<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round" aria-hidden="true" focusable="false"><path d="M4 7.5h2.5L10.5 4v12L6.5 12.5H4z"/><path d="M13.2 7.4a3.5 3.5 0 0 1 0 5.2M15.4 5.3a6.5 6.5 0 0 1 0 9.4"/></svg>';
|
||||||
const originalAutoRead=window.autoReadLastAssistant;
|
const originalAutoRead=window.autoReadLastAssistant;
|
||||||
const originalApplyPreference=window._applyVoiceModePref;
|
const originalApplyPreference=window._applyVoiceModePref;
|
||||||
|
|
||||||
@ -350,12 +368,15 @@
|
|||||||
// Escape peels one layer at a time: an open language menu closes first
|
// Escape peels one layer at a time: an open language menu closes first
|
||||||
// (focus returns to its button), and only a second Escape exits the mode.
|
// (focus returns to its button), and only a second Escape exits the mode.
|
||||||
if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}
|
if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}
|
||||||
|
if(conversation.outMenu&&!conversation.outMenu.hidden){closeOutputMenu(true);return;}
|
||||||
deactivate(true);
|
deactivate(true);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(event.key==='Tab'){
|
if(event.key==='Tab'){
|
||||||
// Minimal focus trap across the overlay controls (language, mute, exit).
|
// Minimal focus trap across the overlay controls (language, output, mute,
|
||||||
const stops=[conversation.langBtn,conversation.muteBtn,conversation.exitBtn].filter(Boolean);
|
// exit); the output button only joins the trap once it is actually shown.
|
||||||
|
const outVisible=conversation.outWrap&&conversation.outWrap.style&&conversation.outWrap.style.display!=='none';
|
||||||
|
const stops=[conversation.langBtn,outVisible?conversation.outBtn:null,conversation.muteBtn,conversation.exitBtn].filter(Boolean);
|
||||||
const current=stops.indexOf(document.activeElement);
|
const current=stops.indexOf(document.activeElement);
|
||||||
const index=current<0?(event.shiftKey?0:stops.length-1):current;
|
const index=current<0?(event.shiftKey?0:stops.length-1):current;
|
||||||
const next=stops[(index+(event.shiftKey?stops.length-1:1))%stops.length];
|
const next=stops[(index+(event.shiftKey?stops.length-1:1))%stops.length];
|
||||||
@ -451,6 +472,118 @@
|
|||||||
return {wrap:wrap,btn:btn,menu:menu,items:items};
|
return {wrap:wrap,btn:btn,menu:menu,items:items};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Output-device selector ────────────────────────────────────────────
|
||||||
|
// A tidy corner control, styled like the language chooser, to pick which audio
|
||||||
|
// output device the voice plays through. Feature-detected (enumerateDevices +
|
||||||
|
// setSinkId); the whole control stays hidden when routing is unsupported or no
|
||||||
|
// output devices are enumerable. Session-only — it never touches storage.
|
||||||
|
function closeOutputMenu(focusButton){
|
||||||
|
if(!conversation||!conversation.outMenu) return;
|
||||||
|
conversation.outMenu.hidden=true;
|
||||||
|
conversation.outMenu.setAttribute('hidden','');
|
||||||
|
if(conversation.outBtn){
|
||||||
|
conversation.outBtn.setAttribute('aria-expanded','false');
|
||||||
|
if(focusButton&&conversation.outBtn.focus) conversation.outBtn.focus();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function openOutputMenu(){
|
||||||
|
if(!conversation||!conversation.outMenu) return;
|
||||||
|
conversation.outMenu.hidden=false;
|
||||||
|
conversation.outMenu.removeAttribute('hidden');
|
||||||
|
if(conversation.outBtn) conversation.outBtn.setAttribute('aria-expanded','true');
|
||||||
|
const items=conversation.outItems||[];
|
||||||
|
const active=items.filter(function(item){return item.getAttribute('aria-checked')==='true';})[0]||items[0];
|
||||||
|
if(active&&active.focus) active.focus();
|
||||||
|
}
|
||||||
|
|
||||||
|
function toggleOutputMenu(){
|
||||||
|
if(!conversation||!conversation.outMenu) return;
|
||||||
|
if(conversation.outMenu.hidden) openOutputMenu(); else closeOutputMenu(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
function outputDeviceLabel(device,index){
|
||||||
|
const label=device&&device.label?String(device.label):'';
|
||||||
|
if(label) return label;
|
||||||
|
return 'Output '+(index+1);
|
||||||
|
}
|
||||||
|
|
||||||
|
function reflectOutputSelection(){
|
||||||
|
if(!conversation||!conversation.outItems) return;
|
||||||
|
conversation.outItems.forEach(function(item){
|
||||||
|
const selected=(item.getAttribute('data-device')||'')===selectedOutputSinkId;
|
||||||
|
item.setAttribute('aria-checked',selected?'true':'false');
|
||||||
|
});
|
||||||
|
if(conversation.outBtn){
|
||||||
|
const chosen=outputDevices.filter(function(device){return device.deviceId===selectedOutputSinkId;})[0];
|
||||||
|
const label=selectedOutputSinkId&&chosen?outputDeviceLabel(chosen,0):'System default';
|
||||||
|
conversation.outBtn.setAttribute('aria-label','Audio output: '+label);
|
||||||
|
conversation.outBtn.setAttribute('title','Output — '+label);
|
||||||
|
if(conversation.outBtn.classList){
|
||||||
|
if(selectedOutputSinkId) conversation.outBtn.classList.add('is-forced');
|
||||||
|
else conversation.outBtn.classList.remove('is-forced');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectOutputDevice(deviceId){
|
||||||
|
selectedOutputSinkId=deviceId||'';
|
||||||
|
reflectOutputSelection();
|
||||||
|
applyOutputSink();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshOutputDevices(){
|
||||||
|
if(!conversation||!conversation.outMenu||!conversation.outWrap) return;
|
||||||
|
if(!outputRoutingSupported()){conversation.outWrap.style.display='none';return;}
|
||||||
|
let devices=[];
|
||||||
|
try{devices=await navigator.mediaDevices.enumerateDevices();}catch(_){devices=[];}
|
||||||
|
// Real, routable outputs only: the empty-deviceId "default" device the
|
||||||
|
// browser reports is represented by our own "System default" entry.
|
||||||
|
outputDevices=(devices||[]).filter(function(device){return device&&device.kind==='audiooutput'&&device.deviceId;});
|
||||||
|
if(!conversation||!conversation.outMenu) return;
|
||||||
|
// Rebuild the menu: a "System default" entry plus every routable output.
|
||||||
|
conversation.outMenu.children=[];
|
||||||
|
const entries=[{deviceId:'',label:'System default'}].concat(
|
||||||
|
outputDevices.map(function(device,index){return {deviceId:device.deviceId,label:outputDeviceLabel(device,index)};})
|
||||||
|
);
|
||||||
|
conversation.outItems=entries.map(function(entry){
|
||||||
|
const item=conversationNode('button','voice-conversation-out-item',{
|
||||||
|
type:'button',role:'menuitemradio','data-device':entry.deviceId,
|
||||||
|
'aria-checked':entry.deviceId===selectedOutputSinkId?'true':'false',
|
||||||
|
});
|
||||||
|
item.textContent=entry.label;
|
||||||
|
item.addEventListener('click',function(){
|
||||||
|
selectOutputDevice(entry.deviceId);
|
||||||
|
closeOutputMenu(true);
|
||||||
|
});
|
||||||
|
conversation.outMenu.appendChild(item);
|
||||||
|
return item;
|
||||||
|
});
|
||||||
|
// Only the default entry means nothing is really selectable — hide it.
|
||||||
|
conversation.outWrap.style.display=outputDevices.length?'':'none';
|
||||||
|
reflectOutputSelection();
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildOutputControl(){
|
||||||
|
const wrap=conversationNode('div','voice-conversation-out');
|
||||||
|
// Hidden until refreshOutputDevices confirms real, routable outputs exist.
|
||||||
|
wrap.style.display='none';
|
||||||
|
const btn=conversationNode('button','voice-conversation-out-btn',{
|
||||||
|
type:'button','aria-haspopup':'menu','aria-expanded':'false',
|
||||||
|
'aria-label':'Audio output: System default',title:'Output — System default',
|
||||||
|
});
|
||||||
|
btn.innerHTML=SPEAKER_ICON_SVG;
|
||||||
|
const menu=conversationNode('div','voice-conversation-out-menu',{role:'menu','aria-label':'Audio output device',hidden:''});
|
||||||
|
menu.hidden=true;
|
||||||
|
btn.addEventListener('click',function(event){
|
||||||
|
if(event&&event.stopPropagation) event.stopPropagation();
|
||||||
|
toggleOutputMenu();
|
||||||
|
});
|
||||||
|
wrap.appendChild(btn);
|
||||||
|
wrap.appendChild(menu);
|
||||||
|
return {wrap:wrap,btn:btn,menu:menu};
|
||||||
|
}
|
||||||
|
|
||||||
function openConversationOverlay(){
|
function openConversationOverlay(){
|
||||||
if(conversation||!conversationUsable()) return;
|
if(conversation||!conversationUsable()) return;
|
||||||
try{
|
try{
|
||||||
@ -485,6 +618,8 @@
|
|||||||
controls.appendChild(exitBtn);
|
controls.appendChild(exitBtn);
|
||||||
const lang=buildLanguageControl();
|
const lang=buildLanguageControl();
|
||||||
root.appendChild(lang.wrap);
|
root.appendChild(lang.wrap);
|
||||||
|
const out=buildOutputControl();
|
||||||
|
root.appendChild(out.wrap);
|
||||||
root.appendChild(orb);
|
root.appendChild(orb);
|
||||||
root.appendChild(stateEl);
|
root.appendChild(stateEl);
|
||||||
root.appendChild(captions);
|
root.appendChild(captions);
|
||||||
@ -494,14 +629,16 @@
|
|||||||
root.addEventListener('keydown',conversationKeydown);
|
root.addEventListener('keydown',conversationKeydown);
|
||||||
// A tap anywhere outside the language menu closes it (never deactivates).
|
// A tap anywhere outside the language menu closes it (never deactivates).
|
||||||
root.addEventListener('click',function(event){
|
root.addEventListener('click',function(event){
|
||||||
if(!conversation||!conversation.langMenu||conversation.langMenu.hidden) return;
|
if(!conversation) return;
|
||||||
const target=event&&event.target;
|
const target=event&&event.target;
|
||||||
const inWrap=target&&typeof target.closest==='function'&&target.closest('.voice-conversation-lang');
|
const closest=target&&typeof target.closest==='function'?target.closest.bind(target):null;
|
||||||
if(!inWrap) closeLanguageMenu(false);
|
if(conversation.langMenu&&!conversation.langMenu.hidden&&!(closest&&closest('.voice-conversation-lang'))) closeLanguageMenu(false);
|
||||||
|
if(conversation.outMenu&&!conversation.outMenu.hidden&&!(closest&&closest('.voice-conversation-out'))) closeOutputMenu(false);
|
||||||
});
|
});
|
||||||
document.body.appendChild(root);
|
document.body.appendChild(root);
|
||||||
conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,langBtn:lang.btn,langMenu:lang.menu,langItems:lang.items,muted:false};
|
conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,langBtn:lang.btn,langMenu:lang.menu,langItems:lang.items,outWrap:out.wrap,outBtn:out.btn,outMenu:out.menu,outItems:[],muted:false};
|
||||||
reflectLanguageSelection();
|
reflectLanguageSelection();
|
||||||
|
refreshOutputDevices();
|
||||||
syncConversationOverlay(state);
|
syncConversationOverlay(state);
|
||||||
if(root.focus) root.focus();
|
if(root.focus) root.focus();
|
||||||
}catch(_){
|
}catch(_){
|
||||||
@ -626,7 +763,9 @@
|
|||||||
if(cue.controller){cue.controller.abort();cue.controller=null;}
|
if(cue.controller){cue.controller.abort();cue.controller=null;}
|
||||||
if(cue.audioWake){const wake=cue.audioWake;cue.audioWake=null;wake();}
|
if(cue.audioWake){const wake=cue.audioWake;cue.audioWake=null;wake();}
|
||||||
if(cue.node){try{cue.node.port.postMessage({type:'cancel'});cue.node.disconnect();}catch(_){ }cue.node=null;}
|
if(cue.node){try{cue.node.port.postMessage({type:'cancel'});cue.node.disconnect();}catch(_){ }cue.node=null;}
|
||||||
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
// The cue shares the ONE playback context; drop the reference but never
|
||||||
|
// close the shared sink here.
|
||||||
|
if(cue.context){cue.context=null;}
|
||||||
cue.gain=null;
|
cue.gain=null;
|
||||||
indicator.classList.remove('is-playing');
|
indicator.classList.remove('is-playing');
|
||||||
indicator.classList.remove('is-ducked');
|
indicator.classList.remove('is-ducked');
|
||||||
@ -722,6 +861,9 @@
|
|||||||
cancelSpeechTurn();
|
cancelSpeechTurn();
|
||||||
releaseMicrophone();
|
releaseMicrophone();
|
||||||
stopPlayback();
|
stopPlayback();
|
||||||
|
closeSharedPlayback();
|
||||||
|
selectedOutputSinkId='';
|
||||||
|
outputDevices=[];
|
||||||
removeConversationOverlay();
|
removeConversationOverlay();
|
||||||
modeBtn.classList.remove('active');
|
modeBtn.classList.remove('active');
|
||||||
setState('error',message);
|
setState('error',message);
|
||||||
@ -786,7 +928,9 @@
|
|||||||
if(playbackSession.node){
|
if(playbackSession.node){
|
||||||
try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ }
|
try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ }
|
||||||
}
|
}
|
||||||
if(playbackSession.context){try{playbackSession.context.close();}catch(_){ }}
|
// The context is the shared spoken-output sink; it is closed only when
|
||||||
|
// hands-free mode ends (closeSharedPlayback), never per playback session.
|
||||||
|
playbackSession.context=null;
|
||||||
playbackSession=null;
|
playbackSession=null;
|
||||||
}
|
}
|
||||||
indicator.classList.remove('is-playing');
|
indicator.classList.remove('is-playing');
|
||||||
@ -817,6 +961,9 @@
|
|||||||
cancelSpeechTurn();
|
cancelSpeechTurn();
|
||||||
releaseMicrophone();
|
releaseMicrophone();
|
||||||
stopPlayback();
|
stopPlayback();
|
||||||
|
closeSharedPlayback();
|
||||||
|
selectedOutputSinkId='';
|
||||||
|
outputDevices=[];
|
||||||
modeBtn.classList.remove('active');
|
modeBtn.classList.remove('active');
|
||||||
setState('idle');
|
setState('idle');
|
||||||
if(showMessage) toast('Hands-free voice mode off');
|
if(showMessage) toast('Hands-free voice mode off');
|
||||||
@ -2015,12 +2162,70 @@
|
|||||||
// Export the deterministic splitter as a narrow diagnostic/test seam.
|
// Export the deterministic splitter as a narrow diagnostic/test seam.
|
||||||
window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;};
|
window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;};
|
||||||
|
|
||||||
|
function outputRoutingSupported(){
|
||||||
|
// The corner output-device selector needs to both ENUMERATE outputs and
|
||||||
|
// ROUTE to one. Feature-detected so unsupported browsers hide it entirely.
|
||||||
|
const Context=window.AudioContext||window.webkitAudioContext;
|
||||||
|
const md=navigator.mediaDevices;
|
||||||
|
const canRoute=(Context&&Context.prototype&&typeof Context.prototype.setSinkId==='function')||
|
||||||
|
(window.HTMLMediaElement&&window.HTMLMediaElement.prototype&&typeof window.HTMLMediaElement.prototype.setSinkId==='function');
|
||||||
|
return !!(md&&typeof md.enumerateDevices==='function'&&canRoute);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyContextSink(context){
|
||||||
|
if(!context||!selectedOutputSinkId||typeof context.setSinkId!=='function') return;
|
||||||
|
try{await context.setSinkId(selectedOutputSinkId);}catch(_){ }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function acquirePlaybackContext(){
|
||||||
|
// One shared AudioContext for ALL spoken output. Its native rate is left to
|
||||||
|
// the browser; the worklet resamples each push from Piper's rate, so cues
|
||||||
|
// and answers (even at different sample rates) share the exact same sink.
|
||||||
|
const Context=window.AudioContext||window.webkitAudioContext;
|
||||||
|
if(!Context) return null;
|
||||||
|
if(sharedPlaybackContext&&sharedPlaybackContext.state!=='closed'){
|
||||||
|
if(sharedPlaybackWorklet){try{await sharedPlaybackWorklet;}catch(_){ }}
|
||||||
|
return sharedPlaybackContext;
|
||||||
|
}
|
||||||
|
let context;
|
||||||
|
try{context=new Context({latencyHint:'interactive'});}catch(_){context=new Context();}
|
||||||
|
sharedPlaybackContext=context;
|
||||||
|
sharedPlaybackWorklet=context.audioWorklet.addModule(WORKLET_URL);
|
||||||
|
try{await sharedPlaybackWorklet;}catch(error){
|
||||||
|
if(sharedPlaybackContext===context){sharedPlaybackContext=null;sharedPlaybackWorklet=null;}
|
||||||
|
try{context.close();}catch(_){ }
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await applyContextSink(context);
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyOutputSink(){
|
||||||
|
// Route the shared context and any in-flight blob element to the chosen sink.
|
||||||
|
applyContextSink(sharedPlaybackContext);
|
||||||
|
if(currentAudio&&typeof currentAudio.setSinkId==='function'){
|
||||||
|
try{currentAudio.setSinkId(selectedOutputSinkId||'');}catch(_){ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeSharedPlayback(){
|
||||||
|
const context=sharedPlaybackContext;
|
||||||
|
sharedPlaybackContext=null;
|
||||||
|
sharedPlaybackWorklet=null;
|
||||||
|
if(context){try{context.close();}catch(_){ }}
|
||||||
|
}
|
||||||
|
|
||||||
function playBlob(blob,token){
|
function playBlob(blob,token){
|
||||||
return new Promise(function(resolve,reject){
|
return new Promise(function(resolve,reject){
|
||||||
if(!active||token!==generation){resolve();return;}
|
if(!active||token!==generation){resolve();return;}
|
||||||
const session=playbackSession;
|
const session=playbackSession;
|
||||||
const url=URL.createObjectURL(blob);
|
const url=URL.createObjectURL(blob);
|
||||||
const audio=new Audio(url);
|
const audio=new Audio(url);
|
||||||
|
// Same chosen output as the PCM path: the WAV fallback element is routed to
|
||||||
|
// the selected sink so it can never split onto a different device.
|
||||||
|
if(selectedOutputSinkId&&typeof audio.setSinkId==='function'){
|
||||||
|
try{audio.setSinkId(selectedOutputSinkId);}catch(_){ }
|
||||||
|
}
|
||||||
currentAudio=audio;
|
currentAudio=audio;
|
||||||
let settled=false;
|
let settled=false;
|
||||||
function cleanup(callback,value){
|
function cleanup(callback,value){
|
||||||
@ -2092,7 +2297,8 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function thinkingCueStillOwned(cue){
|
function thinkingCueStillOwned(cue){
|
||||||
return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking');
|
// Mute-aware: a muted conversation (the user stepped away) never chatters.
|
||||||
|
return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking'&&!(conversation&&conversation.muted));
|
||||||
}
|
}
|
||||||
|
|
||||||
function scheduleNextThinkingCue(cue,delay){
|
function scheduleNextThinkingCue(cue,delay){
|
||||||
@ -2123,10 +2329,10 @@
|
|||||||
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||'22050',10);
|
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||'22050',10);
|
||||||
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000) throw new Error('Cached thinking cue metadata invalid');
|
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000) throw new Error('Cached thinking cue metadata invalid');
|
||||||
if(!thinkingCueStillOwned(cue)) return;
|
if(!thinkingCueStillOwned(cue)) return;
|
||||||
const Context=window.AudioContext||window.webkitAudioContext;
|
// The cue plays through the ONE shared spoken-output sink, exactly like the
|
||||||
try{cue.context=new Context({sampleRate:sampleRate,latencyHint:'interactive'});}catch(_){cue.context=new Context();}
|
// reply — never its own AudioContext (which could route to another device).
|
||||||
await cue.context.audioWorklet.addModule(WORKLET_URL);
|
cue.context=await acquirePlaybackContext();
|
||||||
if(!thinkingCueStillOwned(cue)) return;
|
if(!cue.context||!thinkingCueStillOwned(cue)) return;
|
||||||
cue.node=new AudioWorkletNode(cue.context,'atlas-pcm-playback');
|
cue.node=new AudioWorkletNode(cue.context,'atlas-pcm-playback');
|
||||||
cue.gain=cue.context.createGain();
|
cue.gain=cue.context.createGain();
|
||||||
cue.node.connect(cue.gain);
|
cue.node.connect(cue.gain);
|
||||||
@ -2163,7 +2369,7 @@
|
|||||||
}
|
}
|
||||||
cue.controller=null;
|
cue.controller=null;
|
||||||
if(cue.node){try{cue.node.disconnect();}catch(_){ }cue.node=null;}
|
if(cue.node){try{cue.node.disconnect();}catch(_){ }cue.node=null;}
|
||||||
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
if(cue.context){cue.context=null;}
|
||||||
cue.gain=null;
|
cue.gain=null;
|
||||||
indicator.classList.remove('is-playing');
|
indicator.classList.remove('is-playing');
|
||||||
if(thinkingCueStillOwned(cue)) scheduleNextThinkingCue(cue,THINKING_CUE_INTERVAL_MS);
|
if(thinkingCueStillOwned(cue)) scheduleNextThinkingCue(cue,THINKING_CUE_INTERVAL_MS);
|
||||||
@ -2237,19 +2443,14 @@
|
|||||||
if(session.sampleRate!==asset.sampleRate) throw new Error('Streaming speech sample rate changed mid-turn');
|
if(session.sampleRate!==asset.sampleRate) throw new Error('Streaming speech sample rate changed mid-turn');
|
||||||
return session;
|
return session;
|
||||||
}
|
}
|
||||||
const Context=window.AudioContext||window.webkitAudioContext;
|
// One shared sink for every spoken sound (reply, cues, blob fallback); the
|
||||||
let context;
|
// worklet resamples Piper's rate to the context's native rate. The shared
|
||||||
try{
|
// context is never closed per-session — only when hands-free mode ends.
|
||||||
// Keep Piper at its native rate and let the browser/audio device own any
|
const context=await acquirePlaybackContext();
|
||||||
// final hardware conversion; the worklet interpolator is a fallback.
|
if(!context) return null;
|
||||||
context=new Context({sampleRate:asset.sampleRate,latencyHint:'interactive'});
|
if(!active||token!==generation) return null;
|
||||||
}catch(_){
|
|
||||||
context=new Context();
|
|
||||||
}
|
|
||||||
await context.audioWorklet.addModule(WORKLET_URL);
|
|
||||||
if(!active||token!==generation){context.close();return;}
|
|
||||||
const node=new AudioWorkletNode(context,'atlas-pcm-playback');
|
const node=new AudioWorkletNode(context,'atlas-pcm-playback');
|
||||||
if(!session||session.cancelled){context.close();return;}
|
if(!session||session.cancelled) return null;
|
||||||
const gain=context.createGain();
|
const gain=context.createGain();
|
||||||
session.context=context;
|
session.context=context;
|
||||||
session.node=node;
|
session.node=node;
|
||||||
@ -2334,7 +2535,7 @@
|
|||||||
if(!session||!session.node) return;
|
if(!session||!session.node) return;
|
||||||
await drainPcmPlayback(session,token);
|
await drainPcmPlayback(session,token);
|
||||||
if(session.node){try{session.node.disconnect();}catch(_){ }session.node=null;}
|
if(session.node){try{session.node.disconnect();}catch(_){ }session.node=null;}
|
||||||
if(session.context){try{session.context.close();}catch(_){ }session.context=null;}
|
if(session.context){session.context=null;}
|
||||||
session.gain=null;
|
session.gain=null;
|
||||||
session.drained=null;
|
session.drained=null;
|
||||||
indicator.classList.remove('is-playing');
|
indicator.classList.remove('is-playing');
|
||||||
@ -2468,6 +2669,7 @@
|
|||||||
idleTimer:null,
|
idleTimer:null,
|
||||||
consumed:0,
|
consumed:0,
|
||||||
first:true,
|
first:true,
|
||||||
|
voiceResolved:false,
|
||||||
queue:[],
|
queue:[],
|
||||||
waiters:[],
|
waiters:[],
|
||||||
final:false,
|
final:false,
|
||||||
@ -2477,6 +2679,26 @@
|
|||||||
return speechTurn;
|
return speechTurn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function flushRetainedTail(turn){
|
||||||
|
// An interim message we had BEGUN speaking just stopped being readable — it
|
||||||
|
// folded into the hidden .assistant-segment-worklog-source when the model's
|
||||||
|
// tool call began, so collectAssistantResponse() now returns '' or a
|
||||||
|
// distinct segment. Its retained sourceText still holds an unspoken tail
|
||||||
|
// (turn.consumed < turn.sourceText.length): speak that tail IN FULL, as
|
||||||
|
// final-quality chunks from the text the client already holds — never by
|
||||||
|
// re-reading the now-hidden DOM — so the whole acknowledgement is heard
|
||||||
|
// before the turn transitions to Thinking or a new segment is chunked.
|
||||||
|
// Marking it fully consumed makes a transient empty read that later recovers
|
||||||
|
// the SAME message a no-op instead of a double-speak.
|
||||||
|
if(!turn||typeof turn.sourceText!=='string') return false;
|
||||||
|
const tail=turn.sourceText.slice(turn.consumed).trim();
|
||||||
|
if(!tail){turn.consumed=turn.sourceText.length;return false;}
|
||||||
|
const flushed=adaptiveChunks(tail,true,false);
|
||||||
|
enqueueSpeech(turn,flushed.chunks.length?flushed.chunks:[tail]);
|
||||||
|
turn.consumed=turn.sourceText.length;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
function pumpAssistantResponse(token,isFinal){
|
function pumpAssistantResponse(token,isFinal){
|
||||||
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
||||||
if(isFinal){
|
if(isFinal){
|
||||||
@ -2503,6 +2725,14 @@
|
|||||||
}
|
}
|
||||||
const text=response.text;
|
const text=response.text;
|
||||||
if(!text){
|
if(!text){
|
||||||
|
// The message we had begun speaking just stopped being readable (an
|
||||||
|
// interim acknowledgement folded into the hidden worklog source when the
|
||||||
|
// tool call began). Speak its retained tail in full FIRST, so the
|
||||||
|
// acknowledgement is never truncated and the queue stays non-empty — the
|
||||||
|
// idle fallback can no longer drop to Thinking with an unspoken tail out.
|
||||||
|
// Only for a message we have actually BEGUN speaking (consumed>0): a not-
|
||||||
|
// yet-started message must not be flushed whole on a transient empty read.
|
||||||
|
if(speechTurn&&speechTurn.consumed>0) flushRetainedTail(speechTurn);
|
||||||
if(isFinal){
|
if(isFinal){
|
||||||
// First-sentence-stop guard: once a reply has begun speaking, the
|
// First-sentence-stop guard: once a reply has begun speaking, the
|
||||||
// completion callback can fire a frame before the settle re-render
|
// completion callback can fire a frame before the settle re-render
|
||||||
@ -2538,7 +2768,7 @@
|
|||||||
// of the answer queue.
|
// of the answer queue.
|
||||||
cancelThinkingCues();
|
cancelThinkingCues();
|
||||||
const turn=ensureSpeechTurn(token);
|
const turn=ensureSpeechTurn(token);
|
||||||
if(turn.first){
|
if(!turn.voiceResolved){
|
||||||
// Reply-voice routing, finalized when the first chunk is cut: the
|
// Reply-voice routing, finalized when the first chunk is cut: the
|
||||||
// turn's STT-detected language wins, but script evidence in the reply
|
// turn's STT-detected language wins, but script evidence in the reply
|
||||||
// text corrects a wrong or missing detection (a Spanish reply must
|
// text corrects a wrong or missing detection (a Spanish reply must
|
||||||
@ -2551,14 +2781,33 @@
|
|||||||
const resolved=forcedLanguage||strongReplyLanguage(text)||turn.sttLanguage||detectReplyLanguage(text);
|
const resolved=forcedLanguage||strongReplyLanguage(text)||turn.sttLanguage||detectReplyLanguage(text);
|
||||||
if(resolved) turn.language=resolved;
|
if(resolved) turn.language=resolved;
|
||||||
if(!forcedLanguage&&!turn.sttLanguage&&resolved) sessionLanguage=resolved;
|
if(!forcedLanguage&&!turn.sttLanguage&&resolved) sessionLanguage=resolved;
|
||||||
|
// One audio timeline, one voice: a new logical message after an interim
|
||||||
|
// fold keeps the language already committed to the queue (see the
|
||||||
|
// new-message reset below, which sets turn.first but never re-resolves).
|
||||||
|
turn.voiceResolved=true;
|
||||||
}
|
}
|
||||||
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
|
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
|
||||||
// Renderers can revise the still-unspoken tail. Already-spoken text is
|
const spokenPrefix=turn.sourceText.slice(0,turn.consumed);
|
||||||
// immutable. On the completion callback, advance from the old spoken
|
if(turn.consumed>0&&spokenPrefix&&!text.startsWith(spokenPrefix)){
|
||||||
// offset and close the queue even when the renderer rewrote an earlier
|
// A DISTINCT new message replaced the one we were speaking: the interim
|
||||||
// span; replaying the revised prefix would be more disruptive than
|
// acknowledgement folded to the hidden worklog source and the final
|
||||||
// preserving the already-heard words.
|
// answer rendered as its own segment (the already-spoken interim prefix
|
||||||
if(turn.consumed>0){
|
// is no longer a prefix of what we read). Speak the interim's retained
|
||||||
|
// tail in full, then fall through and process `text` as a BRAND-NEW
|
||||||
|
// message — its chunk offset resets against its OWN text and its audio
|
||||||
|
// queues AFTER the interim drains. Never advance the interim's consumed
|
||||||
|
// offset into the new message's text (that resurfaced the tail out of
|
||||||
|
// order). The voice is kept (turn.voiceResolved), not re-resolved.
|
||||||
|
flushRetainedTail(turn);
|
||||||
|
turn.sourceText='';
|
||||||
|
turn.consumed=0;
|
||||||
|
turn.first=true;
|
||||||
|
}else if(turn.consumed>0){
|
||||||
|
// Same message, unspoken tail revised by the renderer. Already-spoken
|
||||||
|
// text is immutable. On the completion callback, advance from the old
|
||||||
|
// spoken offset and close the queue even when the renderer rewrote an
|
||||||
|
// earlier span; replaying the revised prefix would be more disruptive
|
||||||
|
// than preserving the already-heard words.
|
||||||
if(!isFinal) return;
|
if(!isFinal) return;
|
||||||
thinkingSession=null;
|
thinkingSession=null;
|
||||||
thinkingTurnId='';
|
thinkingTurnId='';
|
||||||
@ -2571,8 +2820,9 @@
|
|||||||
}
|
}
|
||||||
finishSpeechQueue(turn);
|
finishSpeechQueue(turn);
|
||||||
return;
|
return;
|
||||||
|
}else{
|
||||||
|
turn.sourceText='';
|
||||||
}
|
}
|
||||||
turn.sourceText='';
|
|
||||||
}
|
}
|
||||||
turn.sourceText=text;
|
turn.sourceText=text;
|
||||||
const remaining=text.slice(turn.consumed).trimStart();
|
const remaining=text.slice(turn.consumed).trimStart();
|
||||||
@ -2633,6 +2883,8 @@
|
|||||||
clearSttLanguage();
|
clearSttLanguage();
|
||||||
sessionLanguage='';
|
sessionLanguage='';
|
||||||
forcedLanguage='';
|
forcedLanguage='';
|
||||||
|
selectedOutputSinkId='';
|
||||||
|
outputDevices=[];
|
||||||
finalizeAttempts=0;
|
finalizeAttempts=0;
|
||||||
modeBtn.classList.add('active');
|
modeBtn.classList.add('active');
|
||||||
toast('Hands-free private voice mode on');
|
toast('Hands-free private voice mode on');
|
||||||
@ -2719,6 +2971,22 @@
|
|||||||
adaptiveChunks:adaptiveChunks,
|
adaptiveChunks:adaptiveChunks,
|
||||||
sentenceEnd:sentenceEnd,
|
sentenceEnd:sentenceEnd,
|
||||||
cleanForSpeech:cleanForSpeech,
|
cleanForSpeech:cleanForSpeech,
|
||||||
|
// Read-only, behaviour-neutral: lets a deterministic probe observe that the
|
||||||
|
// interim fold never leaves an unspoken tail outstanding while the state
|
||||||
|
// has fallen back to Thinking (the dropped-tail regression).
|
||||||
|
speechTurnSnapshot:function(){
|
||||||
|
if(!speechTurn) return null;
|
||||||
|
return {
|
||||||
|
state:state,
|
||||||
|
consumed:speechTurn.consumed,
|
||||||
|
sourceLength:(speechTurn.sourceText||'').length,
|
||||||
|
queue:speechTurn.queue.slice(),
|
||||||
|
queueLength:speechTurn.queue.length,
|
||||||
|
speakingChunk:speechTurn.speakingChunk||'',
|
||||||
|
waiterLength:speechTurn.waiters.length,
|
||||||
|
final:!!speechTurn.final,
|
||||||
|
};
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -110,6 +110,150 @@ function makeElement(id) {
|
|||||||
return element;
|
return element;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Faithful assistant-turn DOM ─────────────────────────────────────────────
|
||||||
|
// A minimal but real querySelectorAll / closest / getAttribute / hidden /
|
||||||
|
// recursive-textContent implementation, so the interim-acknowledgement FOLD can
|
||||||
|
// be reproduced exactly as the live renderer performs it: the interim segment is
|
||||||
|
// re-tagged .assistant-segment-worklog-source + hidden + aria-hidden, at which
|
||||||
|
// point the script's real collectAssistantResponse() returns '' for it (the same
|
||||||
|
// extraction the response probe locks) and the final answer renders as its own
|
||||||
|
// new .assistant-segment. Supports the compound selectors the extraction uses.
|
||||||
|
function parseDomSelector(selector) {
|
||||||
|
return String(selector).split(',').map((group) => {
|
||||||
|
const term = group.trim();
|
||||||
|
const parts = [];
|
||||||
|
const re = /([.#]?[\w-]+)|\[([\w-]+)(?:([~|^$*]?=)"?([^"\]]*)"?)?\]/g;
|
||||||
|
let m;
|
||||||
|
while ((m = re.exec(term))) {
|
||||||
|
if (m[1]) {
|
||||||
|
if (m[1][0] === '.') parts.push({ kind: 'class', value: m[1].slice(1) });
|
||||||
|
else parts.push({ kind: 'tag', value: m[1].toLowerCase() });
|
||||||
|
} else if (m[2]) {
|
||||||
|
parts.push({ kind: 'attr', name: m[2], op: m[3] || null, value: m[4] });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parts;
|
||||||
|
}).filter((parts) => parts.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
class DomNode {
|
||||||
|
constructor(tag) {
|
||||||
|
this.tag = String(tag || 'div').toLowerCase();
|
||||||
|
this.className = '';
|
||||||
|
this.attributes = new Map();
|
||||||
|
this.children = [];
|
||||||
|
this.parentNode = null;
|
||||||
|
this.hidden = false;
|
||||||
|
this._text = '';
|
||||||
|
const self = this;
|
||||||
|
this.dataset = new Proxy({}, {
|
||||||
|
get(_t, key) {
|
||||||
|
if (typeof key !== 'string') return undefined;
|
||||||
|
const attr = 'data-' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
|
||||||
|
return self.attributes.has(attr) ? self.attributes.get(attr) : undefined;
|
||||||
|
},
|
||||||
|
has(_t, key) {
|
||||||
|
const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
|
||||||
|
return self.attributes.has(attr);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
get classList() {
|
||||||
|
const el = this;
|
||||||
|
return {
|
||||||
|
add(name) { const s = new Set(el.className.split(/\s+/).filter(Boolean)); s.add(name); el.className = [...s].join(' '); },
|
||||||
|
remove(name) { el.className = el.className.split(/\s+/).filter((v) => v && v !== name).join(' '); },
|
||||||
|
contains(name) { return el.className.split(/\s+/).indexOf(name) >= 0; },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
setAttribute(name, value) {
|
||||||
|
this.attributes.set(name, String(value));
|
||||||
|
if (name === 'class') this.className = String(value);
|
||||||
|
if (name === 'hidden') this.hidden = true;
|
||||||
|
}
|
||||||
|
getAttribute(name) {
|
||||||
|
if (name === 'class') return this.className || null;
|
||||||
|
return this.attributes.has(name) ? this.attributes.get(name) : null;
|
||||||
|
}
|
||||||
|
appendChild(child) { child.parentNode = this; this.children.push(child); return child; }
|
||||||
|
set textContent(value) { this._text = String(value); this.children = []; }
|
||||||
|
get textContent() {
|
||||||
|
if (this.children.length) return this.children.map((c) => c.textContent).join('');
|
||||||
|
return this._text;
|
||||||
|
}
|
||||||
|
_matchesTerm(parts) {
|
||||||
|
return parts.every((p) => {
|
||||||
|
if (p.kind === 'class') return this.classList.contains(p.value);
|
||||||
|
if (p.kind === 'tag') return this.tag === p.value;
|
||||||
|
if (p.kind === 'attr') {
|
||||||
|
if (!this.attributes.has(p.name)) return false;
|
||||||
|
return p.op ? this.attributes.get(p.name) === p.value : true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
matches(selector) { return parseDomSelector(selector).some((parts) => this._matchesTerm(parts)); }
|
||||||
|
_walk(out) { for (const c of this.children) { out.push(c); c._walk(out); } return out; }
|
||||||
|
querySelectorAll(selector) {
|
||||||
|
const groups = parseDomSelector(selector);
|
||||||
|
return this._walk([]).filter((node) => groups.some((parts) => node._matchesTerm(parts)));
|
||||||
|
}
|
||||||
|
querySelector(selector) { const all = this.querySelectorAll(selector); return all.length ? all[0] : null; }
|
||||||
|
closest(selector) {
|
||||||
|
const groups = parseDomSelector(selector);
|
||||||
|
let node = this;
|
||||||
|
while (node) { if (groups.some((parts) => node._matchesTerm(parts))) return node; node = node.parentNode; }
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function domEl(tag, className, attrs, text) {
|
||||||
|
const node = new DomNode(tag);
|
||||||
|
if (className) node.setAttribute('class', className);
|
||||||
|
if (attrs) Object.keys(attrs).forEach((k) => node.setAttribute(k, attrs[k]));
|
||||||
|
if (text !== undefined) node.textContent = text;
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A rendered assistant turn: role header, worklog chip, and an assistant-turn
|
||||||
|
// blocks container the interim/final segments hang off of.
|
||||||
|
function buildAssistantTurn() {
|
||||||
|
const turn = domEl('div', 'msg-row assistant-turn', { 'data-role': 'assistant', 'data-session-id': 'session-1' });
|
||||||
|
const role = domEl('div', 'msg-role assistant');
|
||||||
|
role.appendChild(domEl('div', 'role-icon assistant', null, 'H'));
|
||||||
|
role.appendChild(domEl('span', 'msg-role-name', null, 'Hermes'));
|
||||||
|
turn.appendChild(role);
|
||||||
|
const blocks = domEl('div', 'assistant-turn-blocks');
|
||||||
|
turn.appendChild(blocks);
|
||||||
|
turn.blocks = blocks;
|
||||||
|
return turn;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A live (still-streaming) answer segment: no data-raw-text yet, so the script
|
||||||
|
// reads its answer .msg-body — exactly the interim acknowledgement path.
|
||||||
|
function addLiveAnswerSegment(turn, text) {
|
||||||
|
const seg = domEl('div', 'assistant-segment', { 'data-live-assistant': '1' });
|
||||||
|
seg.appendChild(domEl('div', 'msg-body', null, text));
|
||||||
|
turn.blocks.appendChild(seg);
|
||||||
|
return seg;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A settled answer segment carrying the renderer-stamped clean answer text.
|
||||||
|
function addSettledAnswerSegment(turn, text) {
|
||||||
|
const seg = domEl('div', 'assistant-segment', { 'data-raw-text': text });
|
||||||
|
seg.appendChild(domEl('div', 'msg-body', null, text));
|
||||||
|
turn.blocks.appendChild(seg);
|
||||||
|
return seg;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fold an interim segment into the hidden worklog source, exactly as the live
|
||||||
|
// renderer does when a tool call begins: the extraction now excludes it.
|
||||||
|
function foldSegmentIntoWorklog(seg) {
|
||||||
|
seg.setAttribute('class', 'assistant-segment assistant-segment-worklog-source');
|
||||||
|
seg.setAttribute('aria-hidden', 'true');
|
||||||
|
seg.hidden = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Faithful port of StreamingTranscription's endpointing/speculation contract.
|
// Faithful port of StreamingTranscription's endpointing/speculation contract.
|
||||||
class StubSttServer {
|
class StubSttServer {
|
||||||
constructor(words) {
|
constructor(words) {
|
||||||
@ -243,6 +387,7 @@ function makeHarness(options = {}) {
|
|||||||
|
|
||||||
const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' };
|
const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' };
|
||||||
const micTracks = [];
|
const micTracks = [];
|
||||||
|
const sinkCalls = [];
|
||||||
|
|
||||||
const elements = {};
|
const elements = {};
|
||||||
['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg']
|
['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg']
|
||||||
@ -280,6 +425,7 @@ function makeHarness(options = {}) {
|
|||||||
createMediaStreamSource() { return { connect() {}, disconnect() {} }; }
|
createMediaStreamSource() { return { connect() {}, disconnect() {} }; }
|
||||||
createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; }
|
createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; }
|
||||||
resume() { return Promise.resolve(); }
|
resume() { return Promise.resolve(); }
|
||||||
|
setSinkId(id) { sinkCalls.push({ kind: 'context', sink: id }); return Promise.resolve(); }
|
||||||
close() { this.closed = true; return Promise.resolve(); }
|
close() { this.closed = true; return Promise.resolve(); }
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -466,6 +612,7 @@ function makeHarness(options = {}) {
|
|||||||
Audio: function Audio() {
|
Audio: function Audio() {
|
||||||
this.play = () => Promise.resolve();
|
this.play = () => Promise.resolve();
|
||||||
this.pause = () => {};
|
this.pause = () => {};
|
||||||
|
this.setSinkId = (id) => { sinkCalls.push({ kind: 'audio', sink: id }); return Promise.resolve(); };
|
||||||
this.onended = null;
|
this.onended = null;
|
||||||
this.onerror = null;
|
this.onerror = null;
|
||||||
},
|
},
|
||||||
@ -482,6 +629,7 @@ function makeHarness(options = {}) {
|
|||||||
navigator: {
|
navigator: {
|
||||||
mediaDevices: {
|
mediaDevices: {
|
||||||
getSupportedConstraints: () => ({}),
|
getSupportedConstraints: () => ({}),
|
||||||
|
...(options.outputs ? { enumerateDevices: async () => options.outputs } : {}),
|
||||||
getUserMedia: async () => {
|
getUserMedia: async () => {
|
||||||
const track = {
|
const track = {
|
||||||
stop() {},
|
stop() {},
|
||||||
@ -610,6 +758,28 @@ function makeHarness(options = {}) {
|
|||||||
setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; },
|
setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; },
|
||||||
setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; },
|
setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; },
|
||||||
clearAssistantRows() { assistantRows = []; },
|
clearAssistantRows() { assistantRows = []; },
|
||||||
|
// ── Interim-fold reproduction ─────────────────────────────────────────
|
||||||
|
// Stream an interim acknowledgement as a live answer segment, then FOLD it
|
||||||
|
// into the hidden worklog source (as the renderer does at the tool call) and
|
||||||
|
// render the final answer as its own new segment.
|
||||||
|
setInterimAck(text) {
|
||||||
|
const turn = buildAssistantTurn();
|
||||||
|
this._foldTurn = turn;
|
||||||
|
this._interimSeg = addLiveAnswerSegment(turn, text);
|
||||||
|
assistantRows = [turn];
|
||||||
|
},
|
||||||
|
foldInterimAck() { if (this._interimSeg) foldSegmentIntoWorklog(this._interimSeg); },
|
||||||
|
setFinalAnswer(text) {
|
||||||
|
if (!this._foldTurn) return;
|
||||||
|
addSettledAnswerSegment(this._foldTurn, text);
|
||||||
|
},
|
||||||
|
speechTurnSnapshot() {
|
||||||
|
const internals = context.window.__atlasVoiceInternals;
|
||||||
|
return internals && typeof internals.speechTurnSnapshot === 'function'
|
||||||
|
? internals.speechTurnSnapshot() : null;
|
||||||
|
},
|
||||||
|
ttsTexts() { return ttsCalls.map((request) => request.text); },
|
||||||
|
sinkCalls() { return sinkCalls.slice(); },
|
||||||
state() { return elements.voiceModeBar.dataset.voiceState || ''; },
|
state() { return elements.voiceModeBar.dataset.voiceState || ''; },
|
||||||
body() { return bodyElement; },
|
body() { return bodyElement; },
|
||||||
trackStates() { return micTracks.map((track) => track.enabled); },
|
trackStates() { return micTracks.map((track) => track.enabled); },
|
||||||
@ -935,6 +1105,173 @@ scenarios.language_override_forces_stt_and_voice = async () => {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// THE FOLD REGRESSION. A quick spoken acknowledgement streams as an interim
|
||||||
|
// answer segment while a tool call is prepared. The pump chunks the first two
|
||||||
|
// sentences and RETAINS the still-streaming third as an unspoken tail; then the
|
||||||
|
// renderer folds the interim into the hidden .assistant-segment-worklog-source
|
||||||
|
// (extraction → '') and the final answer renders as its own new segment.
|
||||||
|
//
|
||||||
|
// Before the fix: the empty-read branch returned without flushing the retained
|
||||||
|
// tail (the acknowledgement's last words were dropped) and the speech queue
|
||||||
|
// drained with the state fallen back to Thinking while a tail was still
|
||||||
|
// outstanding; the final answer then resurfaced from the interim's stale offset
|
||||||
|
// (a garbled mid-string slice). After the fix: the whole acknowledgement is
|
||||||
|
// spoken, in order, BEFORE the distinct final answer, and the state never falls
|
||||||
|
// back to Thinking with an unspoken interim tail outstanding.
|
||||||
|
scenarios.interim_ack_fold_flushes_tail_before_final = async () => {
|
||||||
|
const harness = makeHarness();
|
||||||
|
// The stub Audio never fires onended, so playback of the first chunk parks and
|
||||||
|
// the queue does not drain (the same reason barge_cut_marker only inspects the
|
||||||
|
// first synthesized chunk). The complete ordered set of chunks that WILL be
|
||||||
|
// spoken is therefore the chunks already handed to synthesis (ttsTexts, in
|
||||||
|
// order) followed by the chunks still queued behind the parked one, assembled
|
||||||
|
// as spokenOrder below. Every assertion is a QUEUE/STATE fact, independent of
|
||||||
|
// the playback stub.
|
||||||
|
const violations = [];
|
||||||
|
const record = () => {
|
||||||
|
const snap = harness.speechTurnSnapshot();
|
||||||
|
if (snap && snap.state === 'thinking' && !snap.final && snap.consumed < snap.sourceLength) {
|
||||||
|
violations.push({ consumed: snap.consumed, sourceLength: snap.sourceLength });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const settle = async (steps) => { for (let i = 0; i < steps; i += 1) { await harness.tick(100); record(); } };
|
||||||
|
|
||||||
|
await harness.start();
|
||||||
|
await harness.silence(300);
|
||||||
|
await harness.speak('alpha', 1300);
|
||||||
|
await harness.silence(1500);
|
||||||
|
await harness.tick(400); // dispatched; thinking; response observer running
|
||||||
|
|
||||||
|
// Interim acknowledgement: two complete sentences plus a third still mid-word,
|
||||||
|
// so the pump enqueues S1+S2 and retains "One moment while I che" unspoken.
|
||||||
|
const interim = 'Sure thing. Let me look that up. One moment while I che';
|
||||||
|
harness.setInterimAck(interim);
|
||||||
|
await settle(4);
|
||||||
|
|
||||||
|
// The tool call begins: fold the interim into the hidden worklog source.
|
||||||
|
harness.foldInterimAck();
|
||||||
|
await settle(6);
|
||||||
|
const afterFold = harness.speechTurnSnapshot();
|
||||||
|
const queuedAfterFold = (afterFold && afterFold.queue) || [];
|
||||||
|
|
||||||
|
// The final answer renders as its own new, distinct segment.
|
||||||
|
const finalAnswer = 'The weather today is sunny and warm. Enjoy your afternoon out there.';
|
||||||
|
harness.setFinalAnswer(finalAnswer);
|
||||||
|
await settle(6);
|
||||||
|
harness.completeResponse();
|
||||||
|
await settle(10);
|
||||||
|
|
||||||
|
const finalSnap = harness.speechTurnSnapshot();
|
||||||
|
const residualQueue = (finalSnap && finalSnap.queue) || [];
|
||||||
|
// The full ordered list of chunks that reach the speech queue for this turn.
|
||||||
|
const spokenOrder = harness.ttsTexts().concat(residualQueue);
|
||||||
|
const joined = spokenOrder.join('');
|
||||||
|
const lastInterimIdx = (() => {
|
||||||
|
let idx = -1;
|
||||||
|
spokenOrder.forEach((t, i) => { if (/Sure thing|look that up|One moment|while I che/.test(t)) idx = i; });
|
||||||
|
return idx;
|
||||||
|
})();
|
||||||
|
const firstFinalIdx = spokenOrder.findIndex((t) => /weather|sunny|afternoon/.test(t));
|
||||||
|
|
||||||
|
return {
|
||||||
|
spokenOrder,
|
||||||
|
// After the fold, the interim's retained tail is the next thing queued, and
|
||||||
|
// it is queued BEFORE any final-answer chunk exists.
|
||||||
|
queuedAfterFold,
|
||||||
|
tailQueuedAfterFold: queuedAfterFold.some((t) => /while I che/.test(t)),
|
||||||
|
noFinalBeforeFold: !queuedAfterFold.some((t) => /weather|sunny|afternoon/.test(t)),
|
||||||
|
// (i) every interim sentence reaches the speech queue: none dropped.
|
||||||
|
interimS1Reached: /Sure thing/.test(joined),
|
||||||
|
interimS2Reached: /look that up/.test(joined),
|
||||||
|
interimTailReached: /while I che/.test(joined),
|
||||||
|
// (iii) the distinct final answer is fully queued too.
|
||||||
|
finalS1Reached: /weather today is sunny/.test(joined),
|
||||||
|
finalS2Reached: /Enjoy your afternoon/.test(joined),
|
||||||
|
// The final answer is queued from its OWN start, never resurfaced from the
|
||||||
|
// interim's stale offset ("nny and warm" is the buggy mid-string slice).
|
||||||
|
finalNotGarbled: !spokenOrder.some((t) => /^nny and warm/.test(t)),
|
||||||
|
// (ii) the interim finishes before the final message's chunks are enqueued.
|
||||||
|
interimBeforeFinal: lastInterimIdx >= 0 && firstFinalIdx >= 0 && lastInterimIdx < firstFinalIdx,
|
||||||
|
// (iv) the state never fell back to Thinking with an unspoken interim tail.
|
||||||
|
thinkingWithUnspokenTail: violations.length,
|
||||||
|
state: harness.state(),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// UNIFIED OUTPUT SINK + device selector. When the browser can enumerate outputs
|
||||||
|
// and setSinkId, a tidy corner control lists them; choosing one routes ALL spoken
|
||||||
|
// output (the reply's blob element here, and — via the shared AudioContext — the
|
||||||
|
// PCM reply and thinking cues) to that single device. When the APIs are missing,
|
||||||
|
// the control hides entirely (a dead control never appears).
|
||||||
|
scenarios.output_device_selector_routes_spoken_output = async () => {
|
||||||
|
const harness = makeHarness({
|
||||||
|
outputs: [
|
||||||
|
{ deviceId: '', kind: 'audiooutput', label: 'System default' },
|
||||||
|
{ deviceId: 'spk-1', kind: 'audiooutput', label: 'Speaker One' },
|
||||||
|
{ deviceId: 'spk-2', kind: 'audiooutput', label: 'Headphones Two' },
|
||||||
|
{ deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await harness.start();
|
||||||
|
await harness.flush();
|
||||||
|
await harness.flush();
|
||||||
|
const overlay = harness.overlay();
|
||||||
|
if (!overlay) return { overlayPresent: false };
|
||||||
|
const outWrap = overlay.children.find(
|
||||||
|
(child) => String(child.className).indexOf('voice-conversation-out') >= 0,
|
||||||
|
);
|
||||||
|
const outBtn = outWrap ? outWrap.children.find(
|
||||||
|
(child) => String(child.className).indexOf('voice-conversation-out-btn') >= 0,
|
||||||
|
) : null;
|
||||||
|
const outMenu = outWrap ? outWrap.children.find(
|
||||||
|
(child) => String(child.className).indexOf('voice-conversation-out-menu') >= 0,
|
||||||
|
) : null;
|
||||||
|
const shownAfterRefresh = outWrap ? outWrap.style.display !== 'none' : false;
|
||||||
|
const items = outMenu ? outMenu.children : [];
|
||||||
|
const itemLabels = items.map((item) => item.textContent);
|
||||||
|
const headphones = items.find((item) => item.getAttribute('data-device') === 'spk-2');
|
||||||
|
if (headphones) headphones.click();
|
||||||
|
const forcedChecked = headphones ? headphones.getAttribute('aria-checked') : null;
|
||||||
|
const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false;
|
||||||
|
// A reply now plays through the blob fallback element, which must be routed to
|
||||||
|
// the chosen sink.
|
||||||
|
await harness.silence(300);
|
||||||
|
await harness.speak('alpha', 1300);
|
||||||
|
await harness.silence(1500);
|
||||||
|
await harness.tick(400);
|
||||||
|
harness.setAssistantReply('A short spoken reply.');
|
||||||
|
harness.completeResponse();
|
||||||
|
await harness.tick(400);
|
||||||
|
const sinks = harness.sinkCalls();
|
||||||
|
return {
|
||||||
|
overlayPresent: true,
|
||||||
|
supported: true,
|
||||||
|
shownAfterRefresh,
|
||||||
|
itemLabels,
|
||||||
|
forcedChecked,
|
||||||
|
btnForced,
|
||||||
|
audioRoutedTo: sinks.filter((c) => c.kind === 'audio').map((c) => c.sink),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// When enumerateDevices/setSinkId are unavailable the selector is present in the
|
||||||
|
// DOM but hidden — never a dead control.
|
||||||
|
scenarios.output_selector_hidden_when_unsupported = async () => {
|
||||||
|
const harness = makeHarness();
|
||||||
|
await harness.start();
|
||||||
|
await harness.flush();
|
||||||
|
const overlay = harness.overlay();
|
||||||
|
if (!overlay) return { overlayPresent: false };
|
||||||
|
const outWrap = overlay.children.find(
|
||||||
|
(child) => String(child.className).indexOf('voice-conversation-out') >= 0,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
overlayPresent: true,
|
||||||
|
controlInDom: !!outWrap,
|
||||||
|
hidden: outWrap ? outWrap.style.display === 'none' : null,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
const output = {};
|
const output = {};
|
||||||
for (const name of Object.keys(scenarios)) {
|
for (const name of Object.keys(scenarios)) {
|
||||||
|
|||||||
@ -361,11 +361,12 @@ def test_chat_voice_uses_private_jetson_services_and_shared_auto_route():
|
|||||||
assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
|
assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
|
||||||
assert "recorder.start();" in voice_script
|
assert "recorder.start();" in voice_script
|
||||||
assert "recorder.start(250)" not in voice_script
|
assert "recorder.start(250)" not in voice_script
|
||||||
|
assert "function flushRetainedTail(turn)" in voice_script # folded interim tail is spoken, never dropped
|
||||||
|
|
||||||
stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
|
stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
|
||||||
assert "def _repetitive_token" in stt_server
|
assert "def _repetitive_token" in stt_server
|
||||||
assert "compression_ratio_threshold=2.0" in stt_server
|
assert '"compression_ratio_threshold": 2.0' in stt_server
|
||||||
assert "no_speech_threshold=0.5" in stt_server
|
assert '"no_speech_threshold": 0.5' in stt_server
|
||||||
|
|
||||||
|
|
||||||
def test_worker_webui_rollouts_do_not_repeat_completed_state_migration():
|
def test_worker_webui_rollouts_do_not_repeat_completed_state_migration():
|
||||||
|
|||||||
@ -77,6 +77,138 @@ def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results):
|
|||||||
assert scenario["micAcquisitions"] == 1
|
assert scenario["micAcquisitions"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_interim_ack_fold_flushes_retained_tail_before_final(probe_results):
|
||||||
|
"""The reported TTS-truncation bug: a quick spoken acknowledgement streams as
|
||||||
|
an interim segment, the pump chunks its first sentences and RETAINS the still-
|
||||||
|
streaming tail; then a tool call folds the interim into the hidden worklog
|
||||||
|
source and the final answer renders as its own segment.
|
||||||
|
|
||||||
|
Before the fix the retained tail was dropped (the empty-read branch returned
|
||||||
|
without flushing it) and the queue drained with the state fallen back to
|
||||||
|
Thinking while a tail was outstanding, and the final answer resurfaced from
|
||||||
|
the interim's stale offset. After the fix the WHOLE acknowledgement reaches
|
||||||
|
the speech queue, in order, before the distinct final answer, and the state
|
||||||
|
never falls back to Thinking with an unspoken interim tail outstanding.
|
||||||
|
"""
|
||||||
|
scenario = probe_results["interim_ack_fold_flushes_tail_before_final"]
|
||||||
|
# (i) every interim sentence reaches the queue — the retained tail is flushed
|
||||||
|
# the moment the segment folds, not dropped.
|
||||||
|
assert scenario["interimS1Reached"] is True
|
||||||
|
assert scenario["interimS2Reached"] is True
|
||||||
|
assert scenario["interimTailReached"] is True, "the folded interim tail was dropped"
|
||||||
|
assert scenario["tailQueuedAfterFold"] is True
|
||||||
|
assert scenario["noFinalBeforeFold"] is True
|
||||||
|
# (ii) the interim finishes before the final message's chunks are enqueued.
|
||||||
|
assert scenario["interimBeforeFinal"] is True
|
||||||
|
# (iii) the distinct final answer is also fully queued, from its own start —
|
||||||
|
# never resurfaced from the interim's stale offset.
|
||||||
|
assert scenario["finalS1Reached"] is True
|
||||||
|
assert scenario["finalS2Reached"] is True
|
||||||
|
assert scenario["finalNotGarbled"] is True
|
||||||
|
assert scenario["spokenOrder"] == [
|
||||||
|
"Sure thing.",
|
||||||
|
"Let me look that up.",
|
||||||
|
"One moment while I che",
|
||||||
|
"The weather today is sunny and warm.",
|
||||||
|
"Enjoy your afternoon out there.",
|
||||||
|
]
|
||||||
|
# (iv) the state never flipped to Thinking with an unspoken interim tail out.
|
||||||
|
assert scenario["thinkingWithUnspokenTail"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_interim_fold_tail_flush_source_contract():
|
||||||
|
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
# The retained tail is flushed as final-quality chunks from the text the
|
||||||
|
# client already holds — never by re-reading the folded/hidden DOM.
|
||||||
|
assert "function flushRetainedTail(turn)" in source
|
||||||
|
assert "const tail=turn.sourceText.slice(turn.consumed).trim();" in source
|
||||||
|
assert "const flushed=adaptiveChunks(tail,true,false);" in source
|
||||||
|
# The empty-read branch flushes the tail before anything else.
|
||||||
|
assert "if(speechTurn&&speechTurn.consumed>0) flushRetainedTail(speechTurn);" in source
|
||||||
|
# A distinct new message after the fold resets the offset against its OWN
|
||||||
|
# text (never advancing the interim's consumed offset into it).
|
||||||
|
assert "const spokenPrefix=turn.sourceText.slice(0,turn.consumed);" in source
|
||||||
|
assert "if(turn.consumed>0&&spokenPrefix&&!text.startsWith(spokenPrefix)){" in source
|
||||||
|
# One audio timeline, one voice: the new message keeps the resolved voice.
|
||||||
|
assert "if(!turn.voiceResolved){" in source
|
||||||
|
assert "turn.voiceResolved=true;" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_device_selector_routes_all_spoken_output(probe_results):
|
||||||
|
"""Unified sink + selector: when the browser can enumerate outputs and
|
||||||
|
setSinkId, the corner control lists the routable outputs and choosing one
|
||||||
|
routes the spoken output (the blob element here) to that device."""
|
||||||
|
scenario = probe_results["output_device_selector_routes_spoken_output"]
|
||||||
|
assert scenario["overlayPresent"] is True
|
||||||
|
assert scenario["shownAfterRefresh"] is True
|
||||||
|
assert scenario["itemLabels"] == ["System default", "Speaker One", "Headphones Two"]
|
||||||
|
# Selection is reflected accessibly and marked as a non-default override.
|
||||||
|
assert scenario["forcedChecked"] == "true"
|
||||||
|
assert scenario["btnForced"] is True
|
||||||
|
# The spoken reply is routed to the chosen sink.
|
||||||
|
assert scenario["audioRoutedTo"] == ["spk-2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_selector_hidden_when_unsupported(probe_results):
|
||||||
|
"""No enumerateDevices/setSinkId: the control is in the DOM but hidden — it
|
||||||
|
never appears as a dead control."""
|
||||||
|
scenario = probe_results["output_selector_hidden_when_unsupported"]
|
||||||
|
assert scenario["overlayPresent"] is True
|
||||||
|
assert scenario["controlInDom"] is True
|
||||||
|
assert scenario["hidden"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_natural_fillers_unified_sink_and_output_selector_source_contract():
|
||||||
|
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||||||
|
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
# (2) Natural spoken fillers per language, with the longer reassurances kept.
|
||||||
|
assert "{id:'umm',text:'Umm.'}" in source
|
||||||
|
assert "{id:'hmm',text:'Hmm.'}" in source
|
||||||
|
assert "{id:'one_sec',text:'One sec.'}" in source
|
||||||
|
assert "{id:'mmm',text:'Mmm.'}" in source
|
||||||
|
assert "{id:'a_ver',text:'A ver.'}" in source
|
||||||
|
assert "{id:'hmm',text:'Хм.'}" in source
|
||||||
|
assert "{id:'sec',text:'Секунду.'}" in source
|
||||||
|
# Answer preempts a filler; a genuine >~1.9s gap gates the first one.
|
||||||
|
assert "const THINKING_CUE_FIRST_MS=1900" in source
|
||||||
|
assert "cancelThinkingCues();\n const turn=ensureSpeechTurn(token)" in source
|
||||||
|
# Mute-aware: a muted conversation never chatters.
|
||||||
|
assert "state==='thinking'&&!(conversation&&conversation.muted)" in source
|
||||||
|
|
||||||
|
# (3) ONE shared sink for every spoken sound. The only AudioContext built for
|
||||||
|
# OUTPUT is inside acquirePlaybackContext; cues and PCM replies both use it.
|
||||||
|
assert "async function acquirePlaybackContext()" in source
|
||||||
|
assert "function closeSharedPlayback()" in source
|
||||||
|
assert "cue.context=await acquirePlaybackContext();" in source
|
||||||
|
assert "const context=await acquirePlaybackContext();" in source
|
||||||
|
# The shared context is never closed per playback session/cue.
|
||||||
|
assert "try{cue.context=new Context(" not in source
|
||||||
|
assert "context=new Context({sampleRate:asset.sampleRate" not in source
|
||||||
|
# The blob fallback element is routed to the same chosen sink.
|
||||||
|
assert "audio.setSinkId(selectedOutputSinkId)" in source
|
||||||
|
|
||||||
|
# The output-device selector: feature-detected, session-only, styled.
|
||||||
|
assert "function outputRoutingSupported()" in source
|
||||||
|
assert "navigator.mediaDevices.enumerateDevices" in source
|
||||||
|
assert "function buildOutputControl()" in source
|
||||||
|
assert "function selectOutputDevice(deviceId)" in source
|
||||||
|
assert "role:'menuitemradio','data-device':entry.deviceId" in source
|
||||||
|
# Session-only: the chosen output never touches storage.
|
||||||
|
out_region = source.split("function buildOutputControl()", 1)[1].split(
|
||||||
|
"function openConversationOverlay", 1
|
||||||
|
)[0]
|
||||||
|
assert "localStorage" not in out_region
|
||||||
|
for token in (
|
||||||
|
".voice-conversation-out",
|
||||||
|
".voice-conversation-out-btn",
|
||||||
|
".voice-conversation-out-menu",
|
||||||
|
".voice-conversation-out-item",
|
||||||
|
):
|
||||||
|
assert token in css
|
||||||
|
|
||||||
|
|
||||||
def test_conversation_overlay_lifecycle(probe_results):
|
def test_conversation_overlay_lifecycle(probe_results):
|
||||||
scenario = probe_results["conversation_overlay_lifecycle"]
|
scenario = probe_results["conversation_overlay_lifecycle"]
|
||||||
assert scenario["overlayPresent"] is True
|
assert scenario["overlayPresent"] is True
|
||||||
@ -308,7 +440,7 @@ def test_conversation_language_selector_source_and_style_contract():
|
|||||||
assert "localStorage" not in source.split("CONVERSATION_LANGUAGES", 1)[1].split("function openConversationOverlay", 1)[0]
|
assert "localStorage" not in source.split("CONVERSATION_LANGUAGES", 1)[1].split("function openConversationOverlay", 1)[0]
|
||||||
# Escape peels the menu before exiting; the language button joins the trap.
|
# Escape peels the menu before exiting; the language button joins the trap.
|
||||||
assert "if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}" in source
|
assert "if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}" in source
|
||||||
assert "[conversation.langBtn,conversation.muteBtn,conversation.exitBtn]" in source
|
assert "[conversation.langBtn,outVisible?conversation.outBtn:null,conversation.muteBtn,conversation.exitBtn]" in source
|
||||||
for token in (".voice-conversation-lang-btn", ".voice-conversation-lang-menu", "menuitemradio"):
|
for token in (".voice-conversation-lang-btn", ".voice-conversation-lang-menu", "menuitemradio"):
|
||||||
pass # menu roles live in JS; assert CSS hooks below
|
pass # menu roles live in JS; assert CSS hooks below
|
||||||
for token in (".voice-conversation-lang", ".voice-conversation-lang-btn", ".voice-conversation-lang-menu", ".voice-conversation-lang-item"):
|
for token in (".voice-conversation-lang", ".voice-conversation-lang-btn", ".voice-conversation-lang-menu", ".voice-conversation-lang-item"):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user