1797 lines
72 KiB
JavaScript
1797 lines
72 KiB
JavaScript
// Natural turn-taking for chat.bstein.dev using the private Jetsons.
|
|
(function(){
|
|
'use strict';
|
|
|
|
const modeBtn=document.getElementById('btnVoiceMode');
|
|
const bar=document.getElementById('voiceModeBar');
|
|
const indicator=document.getElementById('voiceModeIndicator');
|
|
const label=document.getElementById('voiceModeLabel');
|
|
const composer=document.getElementById('msg');
|
|
if(!modeBtn||!bar||!indicator||!label||!composer||!navigator.mediaDevices||!window.MediaRecorder) return;
|
|
|
|
let ready=false;
|
|
let active=false;
|
|
let state='idle';
|
|
let generation=0;
|
|
let turnSequence=0;
|
|
let recorder=null;
|
|
let stream=null;
|
|
let audioContext=null;
|
|
let captureNode=null;
|
|
let vadTimer=null;
|
|
let responsePollTimer=null;
|
|
let currentAudio=null;
|
|
let playbackSession=null;
|
|
let bargeMonitor=null;
|
|
let bargeCancelPromise=null;
|
|
let suppressAutoRead=false;
|
|
let thinkingCue=null;
|
|
let streamingStt=null;
|
|
let captureTurnId='';
|
|
let thinkingTurnId='';
|
|
let thinkingSession=null;
|
|
let assistantBaseline=null;
|
|
let speechTurn=null;
|
|
let errorTimer=null;
|
|
let visualInputLevel=0;
|
|
let streamingCapability={tts:null,stt:null,preflight:null};
|
|
let voicePreflight=null;
|
|
const voiceTabNonce=(function(){
|
|
try{
|
|
const bytes=new Uint8Array(16);
|
|
window.crypto.getRandomValues(bytes);
|
|
return Array.from(bytes,function(value){return value.toString(16).padStart(2,'0');}).join('');
|
|
}catch(_){return '';}
|
|
})();
|
|
const reducedMotion=window.matchMedia?window.matchMedia('(prefers-reduced-motion: reduce)'):{matches:false};
|
|
const ERROR_VISIBLE_MS=3200;
|
|
const STREAMING_CAPABILITY_URL='/api/voice/streaming/capability';
|
|
const TTS_STREAM_URL='/api/tts/stream';
|
|
const STT_STREAM_PATH='/api/transcribe/stream';
|
|
const VOICE_PREFLIGHT_URL='/api/voice/route-preflight';
|
|
const VOICE_PREFLIGHT_DEBOUNCE_MS=200;
|
|
const VOICE_PREFLIGHT_DEADLINE_MS=1100;
|
|
const VOICE_PREFLIGHT_TIERS=['fast','balanced','deep','maximum'];
|
|
const WORKLET_URL='static/atlas-voice-worklet.js';
|
|
const STT_MAX_QUEUED_BYTES=1048576;
|
|
const STT_MAX_ARCHIVE_BYTES=2880000;
|
|
const BARGE_LOOKBACK_MS=900;
|
|
const BARGE_DUCK_FRAMES=2;
|
|
const BARGE_TRIGGER_FRAMES=4;
|
|
const THINKING_CUE_FIRST_MS=1900;
|
|
const THINKING_CUE_INTERVAL_MS=6500;
|
|
const THINKING_CUE_POOLS={
|
|
en:[{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}],
|
|
ru:[{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}],
|
|
es:[{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}],
|
|
};
|
|
const STATE_LABELS={
|
|
listening:'Listening',
|
|
transcribing:'Transcribing…',
|
|
thinking:'Thinking…',
|
|
speaking:'Speaking',
|
|
error:'Voice unavailable',
|
|
idle:'',
|
|
};
|
|
// The only language signal this file trusts is the one the private Whisper
|
|
// service returned for the audio of the turn currently being answered. It is
|
|
// bound to that turn's generation token and consumed exactly once.
|
|
let sttLanguage='';
|
|
let sttLanguageToken=-1;
|
|
const TTS_LANGUAGES=['en','ru','es'];
|
|
const originalAutoRead=window.autoReadLastAssistant;
|
|
const originalApplyPreference=window._applyVoiceModePref;
|
|
|
|
function normalizeSttLanguage(value){
|
|
if(typeof value!=='string') return '';
|
|
const code=value.trim().toLowerCase();
|
|
return TTS_LANGUAGES.indexOf(code)>=0?code:'';
|
|
}
|
|
|
|
function clearSttLanguage(){
|
|
sttLanguage='';
|
|
sttLanguageToken=-1;
|
|
}
|
|
|
|
function rememberSttLanguage(language,token){
|
|
sttLanguage=language||'';
|
|
sttLanguageToken=sttLanguage?token:-1;
|
|
}
|
|
|
|
function takeSttLanguage(token){
|
|
const language=sttLanguageToken===token?sttLanguage:'';
|
|
clearSttLanguage();
|
|
return language;
|
|
}
|
|
|
|
function toast(message){
|
|
if(typeof window.showToast==='function') window.showToast(message,3000);
|
|
}
|
|
|
|
function setState(next,customLabel){
|
|
state=next;
|
|
indicator.className='voice-mode-indicator '+next;
|
|
bar.dataset.voiceState=next;
|
|
bar.setAttribute('aria-busy',next==='transcribing'||next==='thinking'?'true':'false');
|
|
label.textContent=customLabel||STATE_LABELS[next]||'';
|
|
bar.style.display=(active&&next!=='idle')||next==='error'?'':'none';
|
|
resetInputLevel();
|
|
}
|
|
|
|
function resetInputLevel(){
|
|
visualInputLevel=0;
|
|
indicator.style.setProperty('--voice-ripple-scale','1.035');
|
|
indicator.style.setProperty('--voice-ripple-opacity','0.3');
|
|
}
|
|
|
|
function updateInputLevel(rms){
|
|
if(reducedMotion.matches||state!=='listening') return;
|
|
const target=Math.max(0,Math.min(1,(rms-0.01)/0.18));
|
|
visualInputLevel=(visualInputLevel*0.72)+(target*0.28);
|
|
indicator.style.setProperty('--voice-ripple-scale',(1.035+(visualInputLevel*0.16)).toFixed(3));
|
|
indicator.style.setProperty('--voice-ripple-opacity',(0.26+(visualInputLevel*0.48)).toFixed(3));
|
|
}
|
|
|
|
function clearErrorTimer(){
|
|
if(errorTimer){window.clearTimeout(errorTimer);errorTimer=null;}
|
|
}
|
|
|
|
function errorMessage(error,fallback){
|
|
return String((error&&error.message)||fallback||'Voice unavailable').trim();
|
|
}
|
|
|
|
function createAbortController(){
|
|
const Controller=window.AbortController||(typeof AbortController!=='undefined'?AbortController:null);
|
|
return Controller?new Controller():{signal:undefined,abort:function(){ }};
|
|
}
|
|
|
|
function cancelledError(message){
|
|
const error=new Error(message||'Voice turn cancelled');
|
|
error.name='AbortError';
|
|
return error;
|
|
}
|
|
|
|
function cancelSpeechTurn(){
|
|
if(!speechTurn) return;
|
|
speechTurn.cancelled=true;
|
|
speechTurn.final=true;
|
|
while(speechTurn.waiters.length) speechTurn.waiters.shift()(null);
|
|
speechTurn=null;
|
|
}
|
|
|
|
function cancelStreamingStt(){
|
|
if(!streamingStt) return;
|
|
streamingStt.cancel();
|
|
streamingStt=null;
|
|
}
|
|
|
|
function cancelVoicePreflight(turnId){
|
|
const preflight=voicePreflight;
|
|
if(!preflight||(turnId&&preflight.turnId!==turnId)) return;
|
|
voicePreflight=null;
|
|
if(preflight.timer){window.clearTimeout(preflight.timer);preflight.timer=null;}
|
|
if(preflight.deadline){window.clearTimeout(preflight.deadline);preflight.deadline=null;}
|
|
if(preflight.controller){preflight.controller.abort();preflight.controller=null;}
|
|
}
|
|
|
|
function scheduleVoicePreflight(turnId,revision,transcript){
|
|
const capability=streamingCapability.preflight;
|
|
const stable=String(transcript||'').replace(/\s+/g,' ').trim();
|
|
if(!capability||!voiceTabNonce||!active||captureTurnId!==turnId||state!=='listening') return;
|
|
if(!Number.isInteger(revision)||revision<1||stable.length<12||stable.length>512) return;
|
|
if(voicePreflight&&voicePreflight.turnId===turnId&&voicePreflight.transcript===stable) return;
|
|
cancelVoicePreflight();
|
|
const preflight={turnId:turnId,revision:revision,transcript:stable,timer:null,deadline:null,controller:null};
|
|
voicePreflight=preflight;
|
|
preflight.timer=window.setTimeout(async function(){
|
|
preflight.timer=null;
|
|
if(voicePreflight!==preflight||!active||captureTurnId!==turnId||state!=='listening') return;
|
|
const controller=createAbortController();
|
|
preflight.controller=controller;
|
|
preflight.deadline=window.setTimeout(function(){controller.abort();},VOICE_PREFLIGHT_DEADLINE_MS);
|
|
try{
|
|
const response=await fetch(VOICE_PREFLIGHT_URL,{
|
|
method:'POST',
|
|
credentials:'same-origin',
|
|
cache:'no-store',
|
|
headers:{'Content-Type':'application/json'},
|
|
body:JSON.stringify({turn_id:turnId,revision:revision,transcript:stable}),
|
|
signal:controller.signal,
|
|
});
|
|
if(!response.ok) return;
|
|
const advisory=await response.json();
|
|
const tier=advisory&&advisory.tier;
|
|
if(
|
|
voicePreflight!==preflight||!active||captureTurnId!==turnId||state!=='listening'||
|
|
advisory.turn_id!==turnId||advisory.revision!==revision||advisory.advisory!==true||
|
|
VOICE_PREFLIGHT_TIERS.indexOf(tier)<0||advisory.target!=='atlas/auto/'+tier
|
|
) return;
|
|
// This is intentionally feedback-only. The completed transcript still
|
|
// takes the normal Switchyard path and remains the sole routing input.
|
|
const preview=stable.length>60?stable.slice(0,57)+'…':stable;
|
|
label.textContent='Listening · '+preview+' · '+tier;
|
|
}catch(_){
|
|
// A partial can be superseded at any time; advisory failure is silent.
|
|
}finally{
|
|
if(preflight.deadline){window.clearTimeout(preflight.deadline);preflight.deadline=null;}
|
|
preflight.controller=null;
|
|
}
|
|
},VOICE_PREFLIGHT_DEBOUNCE_MS);
|
|
}
|
|
|
|
function cancelThinkingCues(){
|
|
const cue=thinkingCue;
|
|
thinkingCue=null;
|
|
if(!cue) return;
|
|
cue.cancelled=true;
|
|
if(cue.timer){window.clearTimeout(cue.timer);cue.timer=null;}
|
|
if(cue.controller){cue.controller.abort();cue.controller=null;}
|
|
if(cue.audioWake){const wake=cue.audioWake;cue.audioWake=null;wake();}
|
|
if(cue.node){try{cue.node.port.postMessage({type:'cancel'});cue.node.disconnect();}catch(_){ }cue.node=null;}
|
|
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
|
cue.gain=null;
|
|
indicator.classList.remove('is-playing');
|
|
indicator.classList.remove('is-ducked');
|
|
}
|
|
|
|
function clearBargeCancellation(){
|
|
const cancellation=bargeCancelPromise;
|
|
bargeCancelPromise=null;
|
|
if(cancellation&&cancellation.controller) cancellation.controller.abort();
|
|
}
|
|
|
|
function microphoneConstraints(){
|
|
const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
|
|
const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
|
|
if(supported.voiceIsolation) constraints.voiceIsolation=true;
|
|
return constraints;
|
|
}
|
|
|
|
function acquireMicrophone(){
|
|
return navigator.mediaDevices.getUserMedia({audio:microphoneConstraints()});
|
|
}
|
|
|
|
function captureAecIsUsable(capture){
|
|
try{
|
|
const tracks=capture&&capture.getAudioTracks?capture.getAudioTracks():[];
|
|
const settings=tracks.length&&tracks[0].getSettings?tracks[0].getSettings():{};
|
|
return settings.echoCancellation!==false;
|
|
}catch(_){
|
|
return true;
|
|
}
|
|
}
|
|
|
|
window._atlasCaptureAecIsUsable=captureAecIsUsable;
|
|
|
|
function setPlaybackDucked(ducked){
|
|
const session=playbackSession;
|
|
if(session&&session.gain&&session.context){
|
|
const gain=ducked?0.16:1;
|
|
try{session.gain.gain.setTargetAtTime(gain,session.context.currentTime,0.018);}catch(_){session.gain.gain.value=gain;}
|
|
}
|
|
if(currentAudio) currentAudio.volume=ducked?0.16:1;
|
|
const cue=thinkingCue;
|
|
if(cue&&cue.gain&&cue.context){
|
|
const gain=ducked?0.16:1;
|
|
try{cue.gain.gain.setTargetAtTime(gain,cue.context.currentTime,0.018);}catch(_){cue.gain.gain.value=gain;}
|
|
}
|
|
if(ducked) indicator.classList.add('is-ducked'); else indicator.classList.remove('is-ducked');
|
|
}
|
|
|
|
function disposeBargeResources(monitor,keepCapture){
|
|
if(!monitor) return;
|
|
monitor.cancelled=true;
|
|
if(monitor.timer){window.clearInterval(monitor.timer);monitor.timer=null;}
|
|
try{if(monitor.node) monitor.node.disconnect();}catch(_){ }
|
|
try{if(monitor.source) monitor.source.disconnect();}catch(_){ }
|
|
try{if(monitor.silentGain) monitor.silentGain.disconnect();}catch(_){ }
|
|
if(!keepCapture){
|
|
if(monitor.stream) monitor.stream.getTracks().forEach(function(track){track.stop();});
|
|
if(monitor.context){try{monitor.context.close();}catch(_){ }}
|
|
}
|
|
if(!keepCapture) setPlaybackDucked(false);
|
|
}
|
|
|
|
function stopBargeMonitor(){
|
|
const monitor=bargeMonitor;
|
|
bargeMonitor=null;
|
|
disposeBargeResources(monitor,false);
|
|
}
|
|
|
|
function showUnavailable(message){
|
|
generation+=1;
|
|
active=false;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
suppressAutoRead=false;
|
|
clearBargeCancellation();
|
|
stopResponseObserver();
|
|
cancelThinkingCues();
|
|
cancelSpeechTurn();
|
|
stopCapture();
|
|
stopPlayback();
|
|
modeBtn.classList.remove('active');
|
|
setState('error',message);
|
|
clearErrorTimer();
|
|
errorTimer=window.setTimeout(function(){
|
|
errorTimer=null;
|
|
if(!active&&state==='error'&&indicator.classList.contains('error')) setState('idle');
|
|
},ERROR_VISIBLE_MS);
|
|
}
|
|
|
|
function stopCapture(){
|
|
cancelVoicePreflight(captureTurnId);
|
|
stopBargeMonitor();
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
if(recorder&&recorder.state!=='inactive'){
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
recorder=null;
|
|
captureNode=null;
|
|
cancelStreamingStt();
|
|
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
|
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
|
}
|
|
|
|
function stopPlayback(){
|
|
if(currentAudio){
|
|
try{currentAudio.pause();currentAudio.currentTime=0;}catch(_){ }
|
|
currentAudio=null;
|
|
}
|
|
if(playbackSession){
|
|
playbackSession.cancelled=true;
|
|
if(playbackSession.blobWake){playbackSession.blobWake();playbackSession.blobWake=null;}
|
|
playbackSession.controllers.forEach(function(controller){controller.abort();});
|
|
if(playbackSession.lowWaterWake){
|
|
playbackSession.lowWaterWake();
|
|
playbackSession.lowWaterWake=null;
|
|
}
|
|
if(playbackSession.drainWake){playbackSession.drainWake();playbackSession.drainWake=null;}
|
|
if(playbackSession.node){
|
|
try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ }
|
|
}
|
|
if(playbackSession.context){try{playbackSession.context.close();}catch(_){ }}
|
|
playbackSession=null;
|
|
}
|
|
indicator.classList.remove('is-playing');
|
|
indicator.classList.remove('is-ducked');
|
|
}
|
|
|
|
function stopResponseObserver(){
|
|
if(responsePollTimer){clearInterval(responsePollTimer);responsePollTimer=null;}
|
|
}
|
|
|
|
function deactivate(showMessage){
|
|
generation+=1;
|
|
active=false;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
clearErrorTimer();
|
|
clearSttLanguage();
|
|
suppressAutoRead=false;
|
|
clearBargeCancellation();
|
|
stopResponseObserver();
|
|
cancelThinkingCues();
|
|
cancelSpeechTurn();
|
|
stopCapture();
|
|
stopPlayback();
|
|
modeBtn.classList.remove('active');
|
|
setState('idle');
|
|
if(showMessage) toast('Hands-free voice mode off');
|
|
}
|
|
|
|
function restartSoon(token,delay){
|
|
window.setTimeout(function(){
|
|
if(active&&token===generation) startListening(token);
|
|
},delay||500);
|
|
}
|
|
|
|
function assistantRows(){
|
|
return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
|
|
}
|
|
|
|
function readAssistantRow(row){
|
|
if(!row) return '';
|
|
if(row.dataset&&typeof row.dataset.rawText==='string') return row.dataset.rawText;
|
|
return typeof row.textContent==='string'?row.textContent:'';
|
|
}
|
|
|
|
function rememberAssistantBaseline(){
|
|
const rows=assistantRows();
|
|
const row=rows.length?rows[rows.length-1]:null;
|
|
assistantBaseline={row:row,text:readAssistantRow(row),count:rows.length};
|
|
}
|
|
|
|
async function settleBargeCancellation(token){
|
|
const cancellation=bargeCancelPromise;
|
|
if(!cancellation) return true;
|
|
try{await cancellation.promise;}catch(_){ }
|
|
const deadline=Date.now()+10000;
|
|
while(active&&token===generation&&Date.now()<deadline){
|
|
const sessionId=(typeof S!=='undefined'&&S.session)?S.session.session_id:'';
|
|
const activeStreamId=typeof S!=='undefined'?String(S.activeStreamId||''):'';
|
|
const busy=typeof S!=='undefined'&&!!S.busy;
|
|
if(sessionId!==cancellation.sessionId) break;
|
|
if(!busy&&(!cancellation.streamId||activeStreamId!==cancellation.streamId)){
|
|
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
|
|
return true;
|
|
}
|
|
await new Promise(function(resolve){window.setTimeout(resolve,50);});
|
|
}
|
|
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
|
|
return false;
|
|
}
|
|
|
|
async function sendTranscript(transcript,token,language){
|
|
if(!active||token!==generation) return;
|
|
cancelVoicePreflight(captureTurnId);
|
|
const text=String(transcript||'').trim();
|
|
if(!text){clearSttLanguage();restartSoon(token,350);return;}
|
|
composer.value=text;
|
|
if(typeof window.autoResize==='function') window.autoResize();
|
|
setState('thinking');
|
|
startBargeMonitor(token);
|
|
if(!bargeCancelPromise&&typeof S!=='undefined'&&(S.busy||S.activeStreamId)){
|
|
suppressAutoRead=true;
|
|
bargeCancelPromise=cancelActiveModelTurn();
|
|
}
|
|
const cancellationSettled=await settleBargeCancellation(token);
|
|
if(!active||token!==generation) return;
|
|
if(!cancellationSettled){
|
|
toast('The previous response did not stop. Please repeat your interruption.');
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
thinkingTurnId=captureTurnId||String(token)+'-'+String(++turnSequence);
|
|
rememberAssistantBaseline();
|
|
rememberSttLanguage(language,token);
|
|
if(typeof window.send==='function'){
|
|
window.send();
|
|
suppressAutoRead=false;
|
|
startResponseObserver(token);
|
|
scheduleThinkingCues(token,language,thinkingTurnId);
|
|
}
|
|
}
|
|
|
|
function audioExtension(mimeType){
|
|
const normalized=String(mimeType||'').toLowerCase();
|
|
if(normalized.indexOf('wav')>=0) return 'wav';
|
|
if(normalized.indexOf('ogg')>=0) return 'ogg';
|
|
if(normalized.indexOf('mp4')>=0) return 'mp4';
|
|
return 'webm';
|
|
}
|
|
|
|
async function transcribe(blob,token){
|
|
if(!active||token!==generation) return;
|
|
setState('transcribing');
|
|
const ext=audioExtension(blob.type);
|
|
const form=new FormData();
|
|
form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
|
|
try{
|
|
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
|
const payload=await response.json().catch(function(){return {};});
|
|
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
|
sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language));
|
|
}catch(error){
|
|
if(!active||token!==generation) return;
|
|
const message=errorMessage(error,'Private Whisper is unavailable');
|
|
showUnavailable(message);
|
|
toast(message);
|
|
// If the browser supplies its own recognizer, hand control back to the
|
|
// upstream voice implementation until the Jetson becomes healthy again.
|
|
if(window.SpeechRecognition||window.webkitSpeechRecognition){
|
|
modeBtn.removeEventListener('click',onVoiceClick,true);
|
|
window.setTimeout(function(){modeBtn.click();},50);
|
|
}
|
|
}
|
|
}
|
|
|
|
function websocketUrl(path){
|
|
const protocol=window.location&&window.location.protocol==='https:'?'wss:':'ws:';
|
|
return protocol+'//'+window.location.host+path;
|
|
}
|
|
|
|
function encodePcm16(samples){
|
|
const bytes=new ArrayBuffer(samples.length*2);
|
|
const view=new DataView(bytes);
|
|
for(let index=0;index<samples.length;index+=1){
|
|
const value=Math.max(-1,Math.min(1,samples[index]));
|
|
view.setInt16(index*2,value<0?value*32768:value*32767,true);
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
function pcm16WavBlob(buffers,sampleRate){
|
|
const chunks=Array.isArray(buffers)?buffers:[];
|
|
const pcmBytes=chunks.reduce(function(total,chunk){return total+(chunk?chunk.byteLength:0);},0);
|
|
const wav=new ArrayBuffer(44+pcmBytes);
|
|
const view=new DataView(wav);
|
|
const write=function(offset,value){for(let index=0;index<value.length;index+=1)view.setUint8(offset+index,value.charCodeAt(index));};
|
|
write(0,'RIFF');
|
|
view.setUint32(4,36+pcmBytes,true);
|
|
write(8,'WAVE');
|
|
write(12,'fmt ');
|
|
view.setUint32(16,16,true);
|
|
view.setUint16(20,1,true);
|
|
view.setUint16(22,1,true);
|
|
view.setUint32(24,sampleRate,true);
|
|
view.setUint32(28,sampleRate*2,true);
|
|
view.setUint16(32,2,true);
|
|
view.setUint16(34,16,true);
|
|
write(36,'data');
|
|
view.setUint32(40,pcmBytes,true);
|
|
const output=new Uint8Array(wav,44);
|
|
let offset=0;
|
|
chunks.forEach(function(chunk){const bytes=new Uint8Array(chunk);output.set(bytes,offset);offset+=bytes.length;});
|
|
return new Blob([wav],{type:'audio/wav'});
|
|
}
|
|
|
|
window._atlasPcm16WavBlob=pcm16WavBlob;
|
|
|
|
function createResampler(sourceRate,targetRate){
|
|
let tail=null;
|
|
let position=0;
|
|
const ratio=sourceRate/targetRate;
|
|
return function(samples){
|
|
const input=new Float32Array(samples.length+(tail===null?0:1));
|
|
let offset=0;
|
|
if(tail!==null){input[0]=tail;offset=1;}
|
|
input.set(samples,offset);
|
|
if(input.length<2){tail=input.length?input[0]:tail;return new Float32Array(0);}
|
|
const output=[];
|
|
while(position<input.length-1){
|
|
const left=Math.floor(position);
|
|
const fraction=position-left;
|
|
output.push(input[left]+((input[left+1]-input[left])*fraction));
|
|
position+=ratio;
|
|
}
|
|
position-=input.length-1;
|
|
tail=input[input.length-1];
|
|
return new Float32Array(output);
|
|
};
|
|
}
|
|
|
|
function createStreamingSttSession(turnId,context){
|
|
if(!streamingCapability.stt||!window.WebSocket||!window.location) return null;
|
|
let socket;
|
|
let settled=false;
|
|
let committed=false;
|
|
let speculative=false;
|
|
let workletReady=false;
|
|
let partialRevision=-1;
|
|
let lastPreflightText='';
|
|
let queue=[];
|
|
let queuedBytes=0;
|
|
let flushTimer=null;
|
|
let archive=[];
|
|
let archiveBytes=0;
|
|
let cancelled=false;
|
|
let resolveFinal;
|
|
let rejectFinal;
|
|
const finalPromise=new Promise(function(resolve,reject){resolveFinal=resolve;rejectFinal=reject;});
|
|
// A transport can fail before recorder.onstop awaits it. Keep the rejection
|
|
// observed without changing what the later await receives.
|
|
finalPromise.catch(function(){ });
|
|
const resample=createResampler(context.sampleRate,16000);
|
|
|
|
function clearFlushTimer(){
|
|
if(flushTimer){window.clearTimeout(flushTimer);flushTimer=null;}
|
|
}
|
|
function clearQueue(){queue=[];queuedBytes=0;clearFlushTimer();}
|
|
function clearArchive(){archive=[];archiveBytes=0;}
|
|
function archivePcm(bytes){
|
|
if(cancelled||!bytes||!bytes.byteLength||archiveBytes>=STT_MAX_ARCHIVE_BYTES) return;
|
|
const retained=Math.min(bytes.byteLength,STT_MAX_ARCHIVE_BYTES-archiveBytes);
|
|
const even=retained-(retained%2);
|
|
if(!even) return;
|
|
archive.push(bytes.slice(0,even));
|
|
archiveBytes+=even;
|
|
}
|
|
function reject(error){
|
|
if(settled) return;
|
|
settled=true;
|
|
cancelVoicePreflight(turnId);
|
|
clearQueue();
|
|
rejectFinal(error instanceof Error?error:new Error(String(error||'Streaming transcription failed')));
|
|
}
|
|
function sendJson(payload){
|
|
if(socket&&socket.readyState===1){socket.send(JSON.stringify(payload));return true;}
|
|
return false;
|
|
}
|
|
function scheduleFlush(){
|
|
if(flushTimer||settled||!queue.length||!socket||socket.readyState!==1) return;
|
|
flushTimer=window.setTimeout(function(){flushTimer=null;flush();},20);
|
|
}
|
|
function flush(){
|
|
clearFlushTimer();
|
|
if(settled){clearQueue();return;}
|
|
if(!socket||socket.readyState!==1) return;
|
|
while(queue.length&&socket.bufferedAmount<524288){
|
|
const bytes=queue.shift();
|
|
queuedBytes=Math.max(0,queuedBytes-bytes.byteLength);
|
|
socket.send(bytes);
|
|
}
|
|
if(queue.length) scheduleFlush();
|
|
}
|
|
try{
|
|
const csrf=String((window.__HERMES_CONFIG__&&window.__HERMES_CONFIG__.csrfToken)||'');
|
|
const protocols=['hermes-voice-v1'];
|
|
if(csrf) protocols.push('hermes-csrf.'+csrf);
|
|
socket=new WebSocket(websocketUrl(STT_STREAM_PATH),protocols);
|
|
socket.binaryType='arraybuffer';
|
|
}catch(error){
|
|
reject(error);
|
|
return null;
|
|
}
|
|
socket.onopen=function(){
|
|
sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:'auto'});
|
|
flush();
|
|
};
|
|
socket.onmessage=function(event){
|
|
if(typeof event.data!=='string') return;
|
|
let payload;
|
|
try{payload=JSON.parse(event.data);}catch(_){return;}
|
|
if(payload.turn_id!==turnId) return;
|
|
if(payload.type==='partial'&&payload.rolling===true){
|
|
const revision=Number(payload.revision);
|
|
if(Number.isFinite(revision)&&revision>partialRevision){
|
|
partialRevision=revision;
|
|
const stable=String(payload.stable_transcript||'').trim();
|
|
const provisional=String(payload.transcript||'').trim();
|
|
const visible=stable||provisional;
|
|
if(visible&&active&&captureTurnId===turnId&&state==='listening'){
|
|
const preview=visible.length>72?visible.slice(0,69)+'…':visible;
|
|
label.textContent='Listening · '+preview+(stable?'':' · provisional');
|
|
if(stable!==lastPreflightText){
|
|
if(stable){
|
|
lastPreflightText=stable;
|
|
scheduleVoicePreflight(turnId,revision,stable);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}else if(payload.type==='final'&&!settled){
|
|
settled=true;
|
|
cancelVoicePreflight(turnId);
|
|
clearQueue();
|
|
clearArchive();
|
|
resolveFinal({transcript:payload.transcript||'',language:normalizeSttLanguage(payload.language)});
|
|
}else if(payload.type==='error'){
|
|
reject(new Error(payload.error||'Streaming transcription failed'));
|
|
}
|
|
};
|
|
socket.onerror=function(){reject(new Error('Streaming transcription connection failed'));};
|
|
socket.onclose=function(){if(!settled) reject(new Error('Streaming transcription closed before final result'));};
|
|
|
|
return {
|
|
finalPromise:finalPromise,
|
|
setWorkletReady:function(value){workletReady=value;},
|
|
push:function(samples){
|
|
if(cancelled||committed||!workletReady) return;
|
|
const pcm=resample(samples);
|
|
if(!pcm.length) return;
|
|
const bytes=encodePcm16(pcm);
|
|
archivePcm(bytes);
|
|
if(settled) return;
|
|
if(queuedBytes+bytes.byteLength>STT_MAX_QUEUED_BYTES){
|
|
reject(new Error('Streaming transcription backpressure limit exceeded'));
|
|
if(socket&&socket.readyState<2) socket.close(1008,'client backpressure');
|
|
return;
|
|
}
|
|
queue.push(bytes);
|
|
queuedBytes+=bytes.byteLength;
|
|
flush();
|
|
},
|
|
takeFallbackBlob:function(){
|
|
if(!archiveBytes) return null;
|
|
const blob=pcm16WavBlob(archive,16000);
|
|
clearArchive();
|
|
return blob;
|
|
},
|
|
speculate:function(){
|
|
if(settled||committed||speculative) return;
|
|
speculative=true;
|
|
sendJson({type:'speculate',turn_id:turnId});
|
|
},
|
|
resume:function(){
|
|
if(settled||committed||!speculative) return;
|
|
speculative=false;
|
|
sendJson({type:'resume',turn_id:turnId});
|
|
},
|
|
commit:function(){
|
|
if(settled||committed) return finalPromise;
|
|
committed=true;
|
|
flush();
|
|
const commitDeadline=Date.now()+30000;
|
|
const sendCommit=function(){
|
|
if(settled) return;
|
|
if(!socket||socket.readyState>1||Date.now()>=commitDeadline){
|
|
reject(new Error('Streaming transcription closed before commit'));
|
|
return;
|
|
}
|
|
if(socket.readyState!==1||queue.length){flush();window.setTimeout(sendCommit,20);return;}
|
|
if(!sendJson({type:'commit',turn_id:turnId})) reject(new Error('Streaming transcription commit failed'));
|
|
};
|
|
sendCommit();
|
|
return finalPromise;
|
|
},
|
|
cancel:function(){
|
|
cancelled=true;
|
|
cancelVoicePreflight(turnId);
|
|
clearQueue();
|
|
clearArchive();
|
|
if(!settled){sendJson({type:'cancel',turn_id:turnId});reject(new Error('Streaming transcription cancelled'));}
|
|
if(socket&&socket.readyState<2) socket.close(1000,'cancelled');
|
|
},
|
|
};
|
|
}
|
|
|
|
async function installCaptureWorklet(context,source,session){
|
|
if(!session) return false;
|
|
if(!context.audioWorklet||!window.AudioWorkletNode){
|
|
session.cancel();
|
|
if(streamingStt===session) streamingStt=null;
|
|
return false;
|
|
}
|
|
try{
|
|
await context.audioWorklet.addModule(WORKLET_URL);
|
|
if(!streamingStt||streamingStt!==session) return false;
|
|
const node=new AudioWorkletNode(context,'atlas-pcm-capture');
|
|
const silentGain=context.createGain();
|
|
silentGain.gain.value=0;
|
|
let flushResolve=null;
|
|
node.port.onmessage=function(event){
|
|
if(!event.data) return;
|
|
if(event.data.type==='flushed'&&flushResolve){flushResolve();flushResolve=null;return;}
|
|
if(!streamingStt||streamingStt!==session||event.data.type!=='pcm') return;
|
|
session.push(new Float32Array(event.data.samples));
|
|
};
|
|
node._atlasFlush=function(){
|
|
return new Promise(function(resolve){flushResolve=resolve;node.port.postMessage({type:'flush'});});
|
|
};
|
|
source.connect(node);
|
|
node.connect(silentGain);
|
|
silentGain.connect(context.destination);
|
|
captureNode=node;
|
|
session.setWorkletReady(true);
|
|
return true;
|
|
}catch(_){
|
|
session.cancel();
|
|
if(streamingStt===session) streamingStt=null;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function cancelActiveModelTurn(){
|
|
const sessionId=(typeof S!=='undefined'&&S.session)?String(S.session.session_id||''):'';
|
|
const streamId=typeof S!=='undefined'?String(S.activeStreamId||''):'';
|
|
if(!streamId) return {sessionId:sessionId,streamId:'',controller:null,promise:Promise.resolve(false)};
|
|
const controller=createAbortController();
|
|
const timer=window.setTimeout(function(){controller.abort();},1800);
|
|
const promise=(async function(){
|
|
try{
|
|
const url=new URL('api/chat/cancel?stream_id='+encodeURIComponent(streamId),document.baseURI||location.href).href;
|
|
const response=await fetch(url,{credentials:'include',signal:controller.signal});
|
|
let payload=null;
|
|
try{payload=await response.json();}catch(_){ }
|
|
const currentSession=(typeof S!=='undefined'&&S.session)?String(S.session.session_id||''):'';
|
|
if(response.ok&&payload&&payload.cancelled===false&¤tSession===sessionId&&String(S.activeStreamId||'')===streamId){
|
|
S.activeStreamId=null;
|
|
if(S.session) S.session.active_stream_id=null;
|
|
if(typeof setBusy==='function') setBusy(false); else S.busy=false;
|
|
}
|
|
return !!response.ok;
|
|
}catch(_){
|
|
return false;
|
|
}finally{
|
|
window.clearTimeout(timer);
|
|
}
|
|
})();
|
|
return {sessionId:sessionId,streamId:streamId,controller:controller,promise:promise};
|
|
}
|
|
|
|
function trimBargeLookback(monitor){
|
|
const maximum=Math.max(1,Math.round(monitor.context.sampleRate*BARGE_LOOKBACK_MS/1000));
|
|
while(monitor.lookbackFrames>maximum&&monitor.lookback.length){
|
|
const overflow=monitor.lookbackFrames-maximum;
|
|
const first=monitor.lookback[0];
|
|
if(first.length<=overflow){monitor.lookback.shift();monitor.lookbackFrames-=first.length;continue;}
|
|
monitor.lookback[0]=first.slice(overflow);
|
|
monitor.lookbackFrames-=overflow;
|
|
}
|
|
}
|
|
|
|
async function triggerBargeIn(monitor){
|
|
if(!bargeMonitor||bargeMonitor!==monitor||monitor.cancelled||!active||monitor.token!==generation) return;
|
|
bargeMonitor=null;
|
|
if(monitor.timer){window.clearInterval(monitor.timer);monitor.timer=null;}
|
|
// Keep the old capture worklet alive until the new STT worklet is attached.
|
|
// Its lookback becomes an untrimmed handoff buffer so no syllables disappear
|
|
// during AudioWorklet/session setup.
|
|
monitor.handoff=true;
|
|
const oldToken=monitor.token;
|
|
generation+=1;
|
|
const token=generation;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
suppressAutoRead=true;
|
|
clearSttLanguage();
|
|
stopResponseObserver();
|
|
cancelThinkingCues();
|
|
cancelSpeechTurn();
|
|
stopPlayback();
|
|
if(typeof window.stopTTS==='function') window.stopTTS();
|
|
clearBargeCancellation();
|
|
bargeCancelPromise=cancelActiveModelTurn();
|
|
setState('listening','Listening — interrupted');
|
|
if(oldToken===token) return;
|
|
startListening(token,{
|
|
stream:monitor.stream,
|
|
context:monitor.context,
|
|
lookback:monitor.lookback,
|
|
handoffMonitor:monitor,
|
|
heardSpeech:true,
|
|
requireStreamingLookback:true,
|
|
});
|
|
}
|
|
|
|
async function startBargeMonitor(token){
|
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')||bargeMonitor) return;
|
|
const monitor={token:token,cancelled:false,handoff:false,aecUsable:true,timer:null,stream:null,context:null,source:null,node:null,silentGain:null,lookback:[],lookbackFrames:0};
|
|
bargeMonitor=monitor;
|
|
try{
|
|
monitor.stream=await acquireMicrophone();
|
|
if(!active||token!==generation||bargeMonitor!==monitor){disposeBargeResources(monitor,false);return;}
|
|
monitor.aecUsable=captureAecIsUsable(monitor.stream);
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
try{monitor.context=new Context({latencyHint:'interactive'});}catch(_){monitor.context=new Context();}
|
|
const analyser=monitor.context.createAnalyser();
|
|
analyser.fftSize=1024;
|
|
monitor.source=monitor.context.createMediaStreamSource(monitor.stream);
|
|
monitor.source.connect(analyser);
|
|
if(monitor.context.audioWorklet&&window.AudioWorkletNode){
|
|
await monitor.context.audioWorklet.addModule(WORKLET_URL);
|
|
if(bargeMonitor!==monitor||monitor.cancelled){disposeBargeResources(monitor,false);return;}
|
|
monitor.node=new AudioWorkletNode(monitor.context,'atlas-pcm-capture');
|
|
monitor.silentGain=monitor.context.createGain();
|
|
monitor.silentGain.gain.value=0;
|
|
monitor.node.port.onmessage=function(event){
|
|
if(monitor.cancelled||(!monitor.handoff&&bargeMonitor!==monitor)||!event.data||event.data.type!=='pcm') return;
|
|
const samples=new Float32Array(event.data.samples);
|
|
monitor.lookback.push(samples);
|
|
monitor.lookbackFrames+=samples.length;
|
|
if(!monitor.handoff) trimBargeLookback(monitor);
|
|
};
|
|
monitor.source.connect(monitor.node);
|
|
monitor.node.connect(monitor.silentGain);
|
|
monitor.silentGain.connect(monitor.context.destination);
|
|
}
|
|
await monitor.context.resume();
|
|
const samples=new Uint8Array(analyser.fftSize);
|
|
let noiseFloor=0.008;
|
|
let voiceFrames=0;
|
|
let ducked=false;
|
|
let speechArmAt=Date.now()+100;
|
|
let playbackWasActive=false;
|
|
monitor.timer=window.setInterval(function(){
|
|
if(!active||token!==generation||bargeMonitor!==monitor||(state!=='thinking'&&state!=='speaking')){stopBargeMonitor();return;}
|
|
analyser.getByteTimeDomainData(samples);
|
|
let energy=0;
|
|
for(let index=0;index<samples.length;index+=1){const value=(samples[index]-128)/128;energy+=value*value;}
|
|
const rms=Math.sqrt(energy/samples.length);
|
|
const now=Date.now();
|
|
const playbackActive=indicator.classList.contains('is-playing')||!!currentAudio||!!(playbackSession&&playbackSession.node)||!!(thinkingCue&&thinkingCue.node);
|
|
if(playbackActive&&!playbackWasActive){
|
|
// AEC needs a brief convergence window when local speech starts. PCM
|
|
// lookback preserves a real interruption spoken during this guard.
|
|
// A brief cached thinking cue uses the shorter guard so an "um" can
|
|
// still interrupt it; answer playback gets the conservative window.
|
|
speechArmAt=now+(state==='thinking'?150:450);
|
|
voiceFrames=0;
|
|
if(ducked){ducked=false;setPlaybackDucked(false);}
|
|
}else if(!playbackActive&&playbackWasActive){
|
|
speechArmAt=now+100;
|
|
}
|
|
playbackWasActive=playbackActive;
|
|
const threshold=Math.max(0.05,(noiseFloor*3)+0.008);
|
|
// Never treat speaker leakage as an interruption when the browser says
|
|
// the requested AEC was not applied. Quiet Thinking remains fully
|
|
// interruptible, and unknown/unsupported settings preserve normal use.
|
|
const automaticBargeAllowed=!playbackActive||monitor.aecUsable;
|
|
const voiceNow=automaticBargeAllowed&&now>=speechArmAt&&rms>threshold;
|
|
if(!voiceNow) noiseFloor=(noiseFloor*0.975)+(Math.min(rms,threshold)*0.025);
|
|
voiceFrames=voiceNow?Math.min(voiceFrames+1,BARGE_TRIGGER_FRAMES):Math.max(voiceFrames-1,0);
|
|
if(voiceFrames>=BARGE_DUCK_FRAMES&&!ducked){ducked=true;setPlaybackDucked(true);}
|
|
if(!voiceFrames&&ducked){ducked=false;setPlaybackDucked(false);}
|
|
if(voiceFrames>=BARGE_TRIGGER_FRAMES) triggerBargeIn(monitor);
|
|
},50);
|
|
}catch(_){
|
|
if(bargeMonitor===monitor) bargeMonitor=null;
|
|
disposeBargeResources(monitor,false);
|
|
// Barge-in is an optional full-duplex enhancement. The normal finalized
|
|
// microphone turn remains available if the browser rejects concurrent
|
|
// capture or AudioWorklet initialization.
|
|
}
|
|
}
|
|
|
|
async function transcribeStreamingOrFallback(blob,token,session,allowContainerFallback){
|
|
if(!active||token!==generation) return;
|
|
setState('transcribing');
|
|
let pcmFallback=null;
|
|
if(session){
|
|
try{
|
|
const result=await session.commit();
|
|
if(!String(result.transcript||'').trim()) throw new Error('Streaming transcription returned no final text');
|
|
if(active&&token===generation){
|
|
streamingStt=null;
|
|
sendTranscript(result.transcript,token,result.language);
|
|
return;
|
|
}
|
|
}catch(_){
|
|
// The finalized browser container below is the quality-preserving path
|
|
// whenever rolling PCM transport or speculative inference fails.
|
|
if(typeof session.takeFallbackBlob==='function') pcmFallback=session.takeFallbackBlob();
|
|
}
|
|
if(streamingStt===session) streamingStt=null;
|
|
}
|
|
if(pcmFallback){
|
|
transcribe(pcmFallback,token);
|
|
return;
|
|
}
|
|
if(allowContainerFallback===false){
|
|
toast('I missed the start of that interruption. Please repeat it.');
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
transcribe(blob,token);
|
|
}
|
|
|
|
async function startListening(token,reusedCapture){
|
|
if(!active||token!==generation) return;
|
|
stopCapture();
|
|
stopPlayback();
|
|
cancelSpeechTurn();
|
|
clearSttLanguage();
|
|
captureTurnId=(voiceTabNonce?voiceTabNonce+'-':'')+String(token)+'-'+String(++turnSequence);
|
|
setState('listening');
|
|
try{
|
|
const capture=reusedCapture&&reusedCapture.stream?reusedCapture.stream:await acquireMicrophone();
|
|
if(!active||token!==generation){
|
|
if(reusedCapture&&reusedCapture.handoffMonitor) disposeBargeResources(reusedCapture.handoffMonitor,false);
|
|
else capture.getTracks().forEach(function(track){track.stop();});
|
|
return;
|
|
}
|
|
stream=capture;
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
if(reusedCapture&&reusedCapture.context){
|
|
audioContext=reusedCapture.context;
|
|
}else{
|
|
try{
|
|
// Let the browser's native resampler produce Whisper's 16 kHz input.
|
|
audioContext=new Context({sampleRate:16000,latencyHint:'interactive'});
|
|
}catch(_){
|
|
audioContext=new Context();
|
|
}
|
|
}
|
|
const analyser=audioContext.createAnalyser();
|
|
analyser.fftSize=2048;
|
|
const highpass=audioContext.createBiquadFilter();
|
|
highpass.type='highpass';
|
|
highpass.frequency.value=140;
|
|
highpass.Q.value=0.7;
|
|
const mediaSource=audioContext.createMediaStreamSource(stream);
|
|
mediaSource.connect(highpass);
|
|
highpass.connect(analyser);
|
|
streamingStt=createStreamingSttSession(captureTurnId,audioContext);
|
|
// Whisper receives the browser's full-band processed microphone signal;
|
|
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
|
|
// needlessly altered before final recognition.
|
|
if(streamingStt) await installCaptureWorklet(audioContext,mediaSource,streamingStt);
|
|
if(streamingStt&&reusedCapture&&Array.isArray(reusedCapture.lookback)){
|
|
reusedCapture.lookback.forEach(function(samples){streamingStt.push(samples);});
|
|
}
|
|
if(reusedCapture&&reusedCapture.handoffMonitor){
|
|
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
|
reusedCapture.handoffMonitor=null;
|
|
}
|
|
if(reusedCapture&&reusedCapture.requireStreamingLookback&&!streamingStt){
|
|
toast('Streaming transcription is unavailable. Please repeat your interruption.');
|
|
stopCapture();
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
const samples=new Uint8Array(analyser.fftSize);
|
|
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm'];
|
|
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
|
const chunks=[];
|
|
let heardSpeech=!!(reusedCapture&&reusedCapture.heardSpeech);
|
|
let voiceFrames=heardSpeech?3:0;
|
|
let noiseFloor=0.008;
|
|
let lastSpeech=Date.now();
|
|
let speculative=false;
|
|
const started=Date.now();
|
|
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
|
let recordedMime=recorder.mimeType||mime||'';
|
|
recorder.ondataavailable=function(event){
|
|
if(!event.data||!event.data.size) return;
|
|
if(event.data.type) recordedMime=event.data.type;
|
|
chunks.push(event.data);
|
|
};
|
|
recorder.onstop=async function(){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
const finalCaptureNode=captureNode;
|
|
if(finalCaptureNode&&typeof finalCaptureNode._atlasFlush==='function'){
|
|
await Promise.race([
|
|
finalCaptureNode._atlasFlush(),
|
|
new Promise(function(resolve){window.setTimeout(resolve,100);}),
|
|
]);
|
|
}
|
|
const recordedStream=stream;
|
|
stream=null;
|
|
if(recordedStream) recordedStream.getTracks().forEach(function(track){track.stop();});
|
|
const context=audioContext;
|
|
audioContext=null;
|
|
if(context){try{context.close();}catch(_){ }}
|
|
captureNode=null;
|
|
recorder=null;
|
|
if(!active||token!==generation){cancelStreamingStt();return;}
|
|
if(!heardSpeech||!chunks.length){cancelStreamingStt();restartSoon(token,300);return;}
|
|
const session=streamingStt;
|
|
transcribeStreamingOrFallback(
|
|
new Blob(chunks,{type:recordedMime||'audio/webm'}),
|
|
token,
|
|
session,
|
|
!(reusedCapture&&reusedCapture.requireStreamingLookback)
|
|
);
|
|
};
|
|
// Ask the browser for one finalized container at stop. Android Chromium
|
|
// can emit timeslice fragments without a reusable EBML initialization
|
|
// header; concatenating those fragments made otherwise valid recordings
|
|
// intermittently unreadable by ffmpeg. A bounded 90-second Opus capture
|
|
// is small enough to retain as one browser-owned recording.
|
|
recorder.start();
|
|
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1100',10)||1100);
|
|
const speculateMs=Math.min(silenceMs-250,Math.max(450,Math.round(silenceMs*0.55)));
|
|
vadTimer=window.setInterval(function(){
|
|
if(!active||token!==generation||!recorder||recorder.state==='inactive') return;
|
|
analyser.getByteTimeDomainData(samples);
|
|
let energy=0;
|
|
for(let index=0;index<samples.length;index+=1){
|
|
const normalized=(samples[index]-128)/128;
|
|
energy+=normalized*normalized;
|
|
}
|
|
const rms=Math.sqrt(energy/samples.length);
|
|
updateInputLevel(rms);
|
|
const now=Date.now();
|
|
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
|
const voiceNow=rms>speechThreshold;
|
|
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
|
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
|
if(!heardSpeech&&voiceFrames>=3){
|
|
heardSpeech=true;
|
|
lastSpeech=now;
|
|
}else if(heardSpeech&&voiceNow){
|
|
lastSpeech=now;
|
|
if(speculative&&streamingStt){streamingStt.resume();speculative=false;}
|
|
}
|
|
if(heardSpeech&&!voiceNow&&!speculative&&(now-lastSpeech)>=speculateMs&&streamingStt){
|
|
streamingStt.speculate();
|
|
speculative=true;
|
|
}
|
|
const finished=heardSpeech&&(now-lastSpeech)>=silenceMs;
|
|
const timedOut=now-started>=90000;
|
|
const idle=(!heardSpeech)&&(now-started)>=20000;
|
|
if(finished||timedOut||idle){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
},100);
|
|
}catch(error){
|
|
if(reusedCapture&&reusedCapture.handoffMonitor){
|
|
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
|
reusedCapture.handoffMonitor=null;
|
|
}
|
|
if(!active||token!==generation) return;
|
|
const message=errorMessage(error,'Microphone permission is required');
|
|
showUnavailable(message);
|
|
toast(message);
|
|
}
|
|
}
|
|
|
|
function stripHttpUrlsForSpeech(text){
|
|
return String(text||'').replace(/(^|\s+[([{]|\s+|[([{"'])https?:\/\/[^\s<>"']+/gi,function(match,prefix){
|
|
let address=match.slice(prefix.length);
|
|
let suffix='';
|
|
while(address){
|
|
const last=address.slice(-1);
|
|
let trailing=/[.,!?;:…，。！？；：]/.test(last);
|
|
if(last===')') trailing=(address.match(/\)/g)||[]).length>(address.match(/\(/g)||[]).length;
|
|
if(last===']') trailing=(address.match(/\]/g)||[]).length>(address.match(/\[/g)||[]).length;
|
|
if(last==='}') trailing=(address.match(/\}/g)||[]).length>(address.match(/\{/g)||[]).length;
|
|
if(!trailing) break;
|
|
suffix=last+suffix;
|
|
address=address.slice(0,-1);
|
|
}
|
|
const pairs={'(':')','[':']','{':'}'};
|
|
const opening=prefix.slice(-1);
|
|
if(pairs[opening]&&suffix.startsWith(pairs[opening])){
|
|
prefix=prefix.slice(0,-1);
|
|
suffix=suffix.slice(1);
|
|
}
|
|
if(/^\s+$/.test(prefix)&&/^[.,!?;:…，。！？；：]/.test(suffix)) prefix='';
|
|
return prefix+suffix;
|
|
});
|
|
}
|
|
|
|
window._atlasStripHttpUrlsForSpeech=stripHttpUrlsForSpeech;
|
|
|
|
function cleanForSpeech(text){
|
|
const cleaned=typeof window._stripForTTS==='function'?window._stripForTTS(text):String(text||'').replace(/```[\s\S]*?```/g,' code block ');
|
|
return stripHttpUrlsForSpeech(cleaned).replace(/\s+/g,' ').trim();
|
|
}
|
|
|
|
function sentenceEnd(text){
|
|
const source=String(text||'');
|
|
const boundary=/[.!?…]+["')\]}]*(?:\s|$)/g;
|
|
const abbreviation=/(?:\b(?:mr|mrs|ms|dr|prof|sr|jr|st|vs|etc|e\.g|i\.e)|\b[A-Z])\.$/i;
|
|
let match;
|
|
while((match=boundary.exec(source))){
|
|
const end=match.index+match[0].trimEnd().length;
|
|
const prefix=source.slice(0,end).replace(/["')\]}]+$/,'');
|
|
if(prefix.endsWith('.')&&abbreviation.test(prefix)) continue;
|
|
return end;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
function preferredCut(text,min,target,max){
|
|
const bounded=text.slice(0,max);
|
|
const candidates=[];
|
|
const punctuation=/[,;:—–.!?…]+["')\]}]*(?:\s|$)/g;
|
|
let match;
|
|
while((match=punctuation.exec(bounded))){
|
|
const end=match.index+match[0].trimEnd().length;
|
|
if(end>=min) candidates.push({end:end,weight:Math.abs(end-target)});
|
|
}
|
|
if(candidates.length){
|
|
candidates.sort(function(left,right){return left.weight-right.weight;});
|
|
return candidates[0].end;
|
|
}
|
|
const spaces=[];
|
|
const whitespace=/\s+/g;
|
|
while((match=whitespace.exec(bounded))){if(match.index>=min) spaces.push(match.index);}
|
|
if(spaces.length){
|
|
spaces.sort(function(left,right){return Math.abs(left-target)-Math.abs(right-target);});
|
|
return spaces[0];
|
|
}
|
|
return Math.min(max,text.length);
|
|
}
|
|
|
|
function adaptiveChunks(text,final,firstChunk){
|
|
const source=String(text||'').trim();
|
|
if(!source) return {chunks:[],consumed:0};
|
|
const chunks=[];
|
|
let offset=0;
|
|
if(firstChunk!==false){
|
|
const firstSentence=sentenceEnd(source);
|
|
if(firstSentence<0&&!final) return {chunks:[],consumed:0};
|
|
const firstAvailable=firstSentence>=0?firstSentence:source.length;
|
|
const firstSlice=source.slice(0,firstAvailable);
|
|
const firstCut=firstSlice.length<=60?firstSlice.length:preferredCut(firstSlice,40,52,60);
|
|
chunks.push(source.slice(0,firstCut).trim());
|
|
offset=firstCut;
|
|
}
|
|
while(offset<source.length){
|
|
const remaining=source.slice(offset).trimStart();
|
|
offset=source.length-remaining.length;
|
|
if(!remaining) break;
|
|
if(remaining.length<100){
|
|
const complete=sentenceEnd(remaining);
|
|
if(!final&&complete<0) break;
|
|
const end=final?remaining.length:complete;
|
|
chunks.push(remaining.slice(0,end).trim());
|
|
offset+=end;
|
|
continue;
|
|
}
|
|
const cut=preferredCut(remaining,100,120,140);
|
|
if(!final&&cut===remaining.length&&remaining.length<140) break;
|
|
chunks.push(remaining.slice(0,cut).trim());
|
|
offset+=cut;
|
|
}
|
|
return {chunks:chunks.filter(Boolean),consumed:offset};
|
|
}
|
|
|
|
// Export the deterministic splitter as a narrow diagnostic/test seam.
|
|
window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;};
|
|
|
|
function playBlob(blob,token){
|
|
return new Promise(function(resolve,reject){
|
|
if(!active||token!==generation){resolve();return;}
|
|
const session=playbackSession;
|
|
const url=URL.createObjectURL(blob);
|
|
const audio=new Audio(url);
|
|
currentAudio=audio;
|
|
let settled=false;
|
|
function cleanup(callback,value){
|
|
if(settled) return;
|
|
settled=true;
|
|
if(currentAudio===audio) currentAudio=null;
|
|
if(session&&session.blobWake===cancel) session.blobWake=null;
|
|
indicator.classList.remove('is-playing');
|
|
URL.revokeObjectURL(url);
|
|
callback(value);
|
|
}
|
|
function cancel(){cleanup(resolve);}
|
|
if(session) session.blobWake=cancel;
|
|
audio.onended=function(){cleanup(resolve);};
|
|
audio.onerror=function(){cleanup(reject,new Error('Local speech playback failed'));};
|
|
audio.play().then(function(){
|
|
if(active&&token===generation&¤tAudio===audio) indicator.classList.add('is-playing');
|
|
}).catch(function(error){cleanup(reject,error);});
|
|
});
|
|
}
|
|
|
|
function ttsRequest(chunk,language,turnId){
|
|
const request={text:chunk,engine:'atlas',turn_id:turnId};
|
|
if(language) request.language=language;
|
|
return request;
|
|
}
|
|
|
|
async function fetchSpeech(chunk,language,turnId,token){
|
|
// `language` is only ever the private STT result for this turn. When it is
|
|
// absent the field is omitted entirely and the server picks English.
|
|
if(!active||token!==generation) throw cancelledError('Speech turn cancelled');
|
|
const session=playbackSession;
|
|
if(!session||session.cancelled||session.turnId!==turnId) throw cancelledError('Speech turn cancelled');
|
|
const controller=createAbortController();
|
|
session.controllers.add(controller);
|
|
try{
|
|
const response=await fetch('/api/tts',{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json'},
|
|
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
|
signal:controller.signal,
|
|
});
|
|
if(!response.ok){
|
|
const payload=await response.json().catch(function(){return {};});
|
|
throw new Error(payload.error||('Local speech request failed: '+response.status));
|
|
}
|
|
const blob=await response.blob();
|
|
if(!active||token!==generation||session.cancelled||playbackSession!==session) throw cancelledError('Speech turn cancelled');
|
|
return blob;
|
|
}finally{
|
|
session.controllers.delete(controller);
|
|
}
|
|
}
|
|
|
|
function cuePoolOffset(turnId,length){
|
|
let hash=0;
|
|
const source=String(turnId||'');
|
|
for(let index=0;index<source.length;index+=1) hash=((hash*31)+source.charCodeAt(index))>>>0;
|
|
return length?hash%length:0;
|
|
}
|
|
|
|
function thinkingCueStillOwned(cue){
|
|
return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking');
|
|
}
|
|
|
|
function scheduleNextThinkingCue(cue,delay){
|
|
if(!thinkingCueStillOwned(cue)||cue.issued>=cue.pool.length) return;
|
|
cue.timer=window.setTimeout(function(){
|
|
cue.timer=null;
|
|
issueThinkingCue(cue);
|
|
},delay);
|
|
}
|
|
|
|
async function issueThinkingCue(cue){
|
|
if(!thinkingCueStillOwned(cue)||cue.issued>=cue.pool.length) return;
|
|
const entry=cue.pool[(cue.offset+cue.issued)%cue.pool.length];
|
|
const cueNumber=cue.issued+1;
|
|
cue.issued+=1;
|
|
const controller=createAbortController();
|
|
cue.controller=controller;
|
|
try{
|
|
const request=ttsRequest(entry.text,cue.language,cue.turnId+':thinking-cue:'+cueNumber);
|
|
request.cue_id=entry.id;
|
|
const response=await fetch(TTS_STREAM_URL,{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json','Accept':'audio/pcm'},
|
|
body:JSON.stringify(request),
|
|
signal:controller.signal,
|
|
});
|
|
if(!response.ok||!response.body) throw new Error('Cached thinking cue unavailable');
|
|
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||'22050',10);
|
|
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000) throw new Error('Cached thinking cue metadata invalid');
|
|
if(!thinkingCueStillOwned(cue)) return;
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
try{cue.context=new Context({sampleRate:sampleRate,latencyHint:'interactive'});}catch(_){cue.context=new Context();}
|
|
await cue.context.audioWorklet.addModule(WORKLET_URL);
|
|
if(!thinkingCueStillOwned(cue)) return;
|
|
cue.node=new AudioWorkletNode(cue.context,'atlas-pcm-playback');
|
|
cue.gain=cue.context.createGain();
|
|
cue.node.connect(cue.gain);
|
|
cue.gain.connect(cue.context.destination);
|
|
const drained=new Promise(function(resolve){
|
|
cue.audioWake=resolve;
|
|
cue.node.port.onmessage=function(event){
|
|
if((event.data||{}).type==='drained'){
|
|
const wake=cue.audioWake;
|
|
cue.audioWake=null;
|
|
if(wake) wake();
|
|
}
|
|
};
|
|
});
|
|
await cue.context.resume();
|
|
indicator.classList.add('is-playing');
|
|
const reader=response.body.getReader();
|
|
let carry=null;
|
|
while(thinkingCueStillOwned(cue)){
|
|
const result=await reader.read();
|
|
if(result.done) break;
|
|
let bytes=result.value;
|
|
if(carry!==null){const joined=new Uint8Array(bytes.length+1);joined[0]=carry;joined.set(bytes,1);bytes=joined;carry=null;}
|
|
if(bytes.length%2){carry=bytes[bytes.length-1];bytes=bytes.slice(0,-1);}
|
|
if(!bytes.length) continue;
|
|
const copy=bytes.buffer.slice(bytes.byteOffset,bytes.byteOffset+bytes.byteLength);
|
|
cue.node.port.postMessage({type:'push',samples:copy,sampleRate:sampleRate},[copy]);
|
|
}
|
|
if(thinkingCueStillOwned(cue)){
|
|
cue.node.port.postMessage({type:'end'});
|
|
await drained;
|
|
}else{
|
|
try{await reader.cancel();}catch(_){ }
|
|
}
|
|
cue.controller=null;
|
|
if(cue.node){try{cue.node.disconnect();}catch(_){ }cue.node=null;}
|
|
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
|
cue.gain=null;
|
|
indicator.classList.remove('is-playing');
|
|
if(thinkingCueStillOwned(cue)) scheduleNextThinkingCue(cue,THINKING_CUE_INTERVAL_MS);
|
|
}catch(error){
|
|
cue.controller=null;
|
|
if(cue.cancelled||(error&&error.name==='AbortError')) return;
|
|
// Cues are optional. A playback or synthesis failure must not disturb the
|
|
// answer, retry noisily, or disable the primary hands-free conversation.
|
|
if(thinkingCue===cue) cancelThinkingCues();
|
|
}
|
|
}
|
|
|
|
function scheduleThinkingCues(token,language,turnId){
|
|
cancelThinkingCues();
|
|
if(!active||token!==generation||state!=='thinking') return;
|
|
// Missing/unknown is the same fail-safe route as answer TTS: English/Amy.
|
|
// Known STT languages remain bound exactly to their turn voice.
|
|
const localized=normalizeSttLanguage(language)||'en';
|
|
const pool=THINKING_CUE_POOLS[localized];
|
|
const cue={
|
|
token:token,
|
|
turnId:String(turnId||token),
|
|
language:localized,
|
|
pool:pool,
|
|
offset:cuePoolOffset(turnId,pool.length),
|
|
issued:0,
|
|
timer:null,
|
|
controller:null,
|
|
audioWake:null,
|
|
context:null,
|
|
node:null,
|
|
gain:null,
|
|
cancelled:false,
|
|
};
|
|
thinkingCue=cue;
|
|
scheduleNextThinkingCue(cue,THINKING_CUE_FIRST_MS);
|
|
}
|
|
|
|
async function prepareSpeech(chunk,language,turnId,token){
|
|
if(streamingCapability.tts&&window.ReadableStream&&window.AudioWorkletNode){
|
|
const controller=createAbortController();
|
|
if(playbackSession) playbackSession.controllers.add(controller);
|
|
try{
|
|
const response=await fetch(TTS_STREAM_URL,{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json','Accept':'audio/pcm'},
|
|
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
|
signal:controller.signal,
|
|
});
|
|
if(!response.ok||!response.body) throw new Error('Streaming speech unavailable');
|
|
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||String(streamingCapability.tts.sample_rate||22050),10);
|
|
const channels=parseInt(response.headers.get('X-Audio-Channels')||'1',10);
|
|
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000||channels!==1) throw new Error('Unsupported streaming speech format');
|
|
return {kind:'pcm',response:response,sampleRate:sampleRate,controller:controller,token:token,chunk:chunk,language:language,turnId:turnId,started:false};
|
|
}catch(error){
|
|
if(playbackSession) playbackSession.controllers.delete(controller);
|
|
if(error&&error.name==='AbortError') throw error;
|
|
// Disable only this optional transport for the browser session. The
|
|
// complete WAV endpoint preserves voice quality and availability.
|
|
streamingCapability.tts=null;
|
|
}
|
|
}
|
|
return {kind:'blob',blob:await fetchSpeech(chunk,language,turnId,token)};
|
|
}
|
|
|
|
async function ensurePcmPlayback(asset,token){
|
|
if(!active||token!==generation) return null;
|
|
const session=playbackSession;
|
|
if(!session||session.cancelled) return null;
|
|
if(session.node){
|
|
if(session.sampleRate!==asset.sampleRate) throw new Error('Streaming speech sample rate changed mid-turn');
|
|
return session;
|
|
}
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
let context;
|
|
try{
|
|
// Keep Piper at its native rate and let the browser/audio device own any
|
|
// final hardware conversion; the worklet interpolator is a fallback.
|
|
context=new Context({sampleRate:asset.sampleRate,latencyHint:'interactive'});
|
|
}catch(_){
|
|
context=new Context();
|
|
}
|
|
await context.audioWorklet.addModule(WORKLET_URL);
|
|
if(!active||token!==generation){context.close();return;}
|
|
const node=new AudioWorkletNode(context,'atlas-pcm-playback');
|
|
if(!session||session.cancelled){context.close();return;}
|
|
const gain=context.createGain();
|
|
session.context=context;
|
|
session.node=node;
|
|
session.gain=gain;
|
|
session.sampleRate=asset.sampleRate;
|
|
session.bufferedFrames=0;
|
|
session.playbackEnded=false;
|
|
node.connect(gain);
|
|
gain.connect(context.destination);
|
|
await context.resume();
|
|
session.drained=new Promise(function(resolve,reject){
|
|
session.drainWake=resolve;
|
|
session.drainReject=reject;
|
|
});
|
|
indicator.classList.add('is-playing');
|
|
node.port.onmessage=function(event){
|
|
const data=event.data||{};
|
|
if(data.type==='buffer'){
|
|
session.bufferedFrames=data.frames||0;
|
|
if(session.bufferedFrames<session.sampleRate&&session.lowWaterWake){
|
|
const wake=session.lowWaterWake;
|
|
session.lowWaterWake=null;
|
|
wake();
|
|
}
|
|
}else if(data.type==='drained'){
|
|
session.playbackEnded=true;
|
|
const wake=session.drainWake;
|
|
session.drainWake=null;
|
|
if(wake) wake();
|
|
}else if(data.type==='error'){
|
|
const reject=session.drainReject;
|
|
session.drainReject=null;
|
|
if(reject) reject(new Error(data.error||'Streaming speech playback failed'));
|
|
}
|
|
};
|
|
return session;
|
|
}
|
|
|
|
async function playPcm(asset,token){
|
|
const session=await ensurePcmPlayback(asset,token);
|
|
if(!session) return;
|
|
const reader=asset.response.body.getReader();
|
|
let carry=null;
|
|
try{
|
|
while(active&&token===generation&&!session.cancelled){
|
|
if(session.bufferedFrames>asset.sampleRate*2){
|
|
await new Promise(function(resolve){
|
|
let completed=false;
|
|
const wake=function(){if(completed)return;completed=true;session.lowWaterWake=null;resolve();};
|
|
session.lowWaterWake=wake;
|
|
window.setTimeout(wake,3000);
|
|
});
|
|
}
|
|
const result=await reader.read();
|
|
if(result.done) break;
|
|
let bytes=result.value;
|
|
if(carry!==null){
|
|
const joined=new Uint8Array(bytes.length+1);
|
|
joined[0]=carry;joined.set(bytes,1);bytes=joined;carry=null;
|
|
}
|
|
if(bytes.length%2){carry=bytes[bytes.length-1];bytes=bytes.slice(0,-1);}
|
|
if(!bytes.length) continue;
|
|
const copy=bytes.buffer.slice(bytes.byteOffset,bytes.byteOffset+bytes.byteLength);
|
|
session.bufferedFrames+=bytes.byteLength/2;
|
|
asset.started=true;
|
|
session.node.port.postMessage({type:'push',samples:copy,sampleRate:asset.sampleRate},[copy]);
|
|
}
|
|
}finally{
|
|
if((!active||token!==generation||session.cancelled)){try{await reader.cancel();}catch(_){ }}
|
|
session.controllers.delete(asset.controller);
|
|
}
|
|
}
|
|
|
|
async function drainPcmPlayback(session,token){
|
|
if(!session||!session.node||session.cancelled||!active||token!==generation) return;
|
|
if(session.playbackEnded) return;
|
|
session.node.port.postMessage({type:'end'});
|
|
await session.drained;
|
|
}
|
|
|
|
async function closePcmBeforeBlob(session,token){
|
|
if(!session||!session.node) return;
|
|
await drainPcmPlayback(session,token);
|
|
if(session.node){try{session.node.disconnect();}catch(_){ }session.node=null;}
|
|
if(session.context){try{session.context.close();}catch(_){ }session.context=null;}
|
|
session.gain=null;
|
|
session.drained=null;
|
|
indicator.classList.remove('is-playing');
|
|
}
|
|
|
|
async function playPrepared(asset,token){
|
|
if(asset.kind!=='pcm'){
|
|
if(playbackSession&&playbackSession.node) await closePcmBeforeBlob(playbackSession,token);
|
|
return playBlob(asset.blob,token);
|
|
}
|
|
try{
|
|
return await playPcm(asset,token);
|
|
}catch(error){
|
|
if(asset.started) throw error;
|
|
streamingCapability.tts=null;
|
|
if(playbackSession&&playbackSession.node) await closePcmBeforeBlob(playbackSession,token);
|
|
return playBlob(await fetchSpeech(asset.chunk,asset.language,asset.turnId,token),token);
|
|
}
|
|
}
|
|
|
|
function nextSpeechChunk(turn){
|
|
if(turn.cancelled) return Promise.resolve(null);
|
|
if(turn.queue.length) return Promise.resolve(turn.queue.shift());
|
|
if(turn.final) return Promise.resolve(null);
|
|
return new Promise(function(resolve){turn.waiters.push(resolve);});
|
|
}
|
|
|
|
function enqueueSpeech(turn,chunks){
|
|
chunks.forEach(function(chunk){
|
|
if(!chunk) return;
|
|
if(turn.waiters.length) turn.waiters.shift()(chunk); else turn.queue.push(chunk);
|
|
});
|
|
}
|
|
|
|
function finishSpeechQueue(turn){
|
|
turn.final=true;
|
|
while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null);
|
|
}
|
|
|
|
async function runSpeechQueue(turn){
|
|
if(turn.running) return;
|
|
turn.running=true;
|
|
playbackSession={turnId:turn.turnId,controllers:new Set(),context:null,node:null,gain:null,sampleRate:0,bufferedFrames:0,drained:null,drainReject:null,playbackEnded:false,cancelled:false,lowWaterWake:null,drainWake:null,blobWake:null};
|
|
const session=playbackSession;
|
|
try{
|
|
let chunk=await nextSpeechChunk(turn);
|
|
let current=chunk?prepareSpeech(chunk,turn.language,turn.turnId,turn.token):null;
|
|
while(current){
|
|
const asset=await current;
|
|
// At most one later synthesis request exists while this asset plays.
|
|
// If streaming has not produced it yet, the waiter starts it the
|
|
// instant a punctuation-safe chunk arrives.
|
|
const next=nextSpeechChunk(turn);
|
|
const nextPrepared=next.then(function(value){
|
|
return value?prepareSpeech(value,turn.language,turn.turnId,turn.token):null;
|
|
});
|
|
// Barge-in can abort both the playing request and its one-ahead request.
|
|
// Observe the latter even when the cancelled current turn returns first.
|
|
nextPrepared.catch(function(){ });
|
|
await playPrepared(asset,turn.token);
|
|
if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return;
|
|
current=await nextPrepared;
|
|
}
|
|
await drainPcmPlayback(session,turn.token);
|
|
if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300);
|
|
}catch(error){
|
|
if(active&&turn.token===generation&&!turn.cancelled){
|
|
const message=errorMessage(error,'Local speech is unavailable');
|
|
setState('error',message);
|
|
toast(message);
|
|
restartSoon(turn.token,500);
|
|
}
|
|
}finally{
|
|
if(playbackSession===session) stopPlayback();
|
|
if(speechTurn===turn) speechTurn=null;
|
|
}
|
|
}
|
|
|
|
function currentAssistantText(){
|
|
const rows=assistantRows();
|
|
if(!rows.length) return '';
|
|
const row=rows[rows.length-1];
|
|
const text=readAssistantRow(row);
|
|
if(assistantBaseline&&row===assistantBaseline.row&&text===assistantBaseline.text) return '';
|
|
if(assistantBaseline&&rows.length<assistantBaseline.count) return '';
|
|
return cleanForSpeech(text);
|
|
}
|
|
|
|
function ensureSpeechTurn(token){
|
|
if(speechTurn) return speechTurn;
|
|
speechTurn={
|
|
token:token,
|
|
turnId:thinkingTurnId||String(token)+'-'+String(++turnSequence),
|
|
language:takeSttLanguage(token),
|
|
sourceText:'',
|
|
consumed:0,
|
|
first:true,
|
|
queue:[],
|
|
waiters:[],
|
|
final:false,
|
|
cancelled:false,
|
|
running:false,
|
|
};
|
|
return speechTurn;
|
|
}
|
|
|
|
function pumpAssistantResponse(token,isFinal){
|
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
|
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
clearSttLanguage();
|
|
stopResponseObserver();
|
|
cancelThinkingCues();
|
|
cancelSpeechTurn();
|
|
stopPlayback();
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
const text=currentAssistantText();
|
|
if(!text){
|
|
if(isFinal){thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();cancelThinkingCues();restartSoon(token,250);}
|
|
return;
|
|
}
|
|
// Any answer text, even a not-yet-speakable partial clause, owns the audio
|
|
// timeline from this point forward. A cue must never overlap or become part
|
|
// of the answer queue.
|
|
cancelThinkingCues();
|
|
const turn=ensureSpeechTurn(token);
|
|
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
|
|
// Renderers can revise the still-unspoken tail. Already-spoken text is
|
|
// immutable. On the completion callback, advance from the old spoken
|
|
// offset and close the queue even when the renderer rewrote an earlier
|
|
// span; replaying the revised prefix would be more disruptive than
|
|
// preserving the already-heard words.
|
|
if(turn.consumed>0){
|
|
if(!isFinal) return;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
stopResponseObserver();
|
|
const revisedTail=text.slice(Math.min(turn.consumed,text.length)).trim();
|
|
if(revisedTail){
|
|
const revised=adaptiveChunks(revisedTail,true,false);
|
|
enqueueSpeech(turn,revised.chunks.length?revised.chunks:[revisedTail]);
|
|
turn.consumed=text.length;
|
|
}
|
|
finishSpeechQueue(turn);
|
|
return;
|
|
}
|
|
turn.sourceText='';
|
|
}
|
|
turn.sourceText=text;
|
|
const remaining=text.slice(turn.consumed).trimStart();
|
|
const skipped=text.slice(turn.consumed).length-remaining.length;
|
|
const extracted=adaptiveChunks(remaining,!!isFinal,turn.first);
|
|
if(extracted.chunks.length){
|
|
turn.first=false;
|
|
turn.consumed+=skipped+extracted.consumed;
|
|
enqueueSpeech(turn,extracted.chunks);
|
|
setState('speaking');
|
|
runSpeechQueue(turn);
|
|
}
|
|
if(isFinal){
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
stopResponseObserver();
|
|
const tail=text.slice(turn.consumed).trim();
|
|
if(tail){enqueueSpeech(turn,[tail]);turn.consumed=text.length;}
|
|
finishSpeechQueue(turn);
|
|
}
|
|
}
|
|
|
|
function startResponseObserver(token){
|
|
stopResponseObserver();
|
|
// A bounded poll is deliberately used in addition to the upstream
|
|
// completion callback. It sees data-raw-text while SSE is still appending,
|
|
// so the first complete sentence can reach Piper before generation ends.
|
|
responsePollTimer=window.setInterval(function(){pumpAssistantResponse(token,false);},75);
|
|
}
|
|
|
|
function speakResponse(token){
|
|
pumpAssistantResponse(token,true);
|
|
}
|
|
|
|
function validStreamingCapability(payload){
|
|
if(!payload||typeof payload!=='object') return {tts:null,stt:null,preflight:null};
|
|
const tts=payload.tts&&payload.tts.available===true&&payload.tts.transport==='http'&&payload.tts.format==='pcm_s16le'?payload.tts:null;
|
|
const stt=payload.stt&&payload.stt.available===true&&payload.stt.transport==='websocket'&&payload.stt.format==='pcm_s16le'&&payload.stt.sample_rate===16000?payload.stt:null;
|
|
const preflight=payload.preflight&&payload.preflight.available===true&&payload.preflight.path===VOICE_PREFLIGHT_URL&&payload.preflight.advisory===true?payload.preflight:null;
|
|
return {tts:tts,stt:stt,preflight:preflight};
|
|
}
|
|
|
|
async function discoverStreamingCapability(){
|
|
try{
|
|
const response=await fetch(STREAMING_CAPABILITY_URL,{cache:'no-store'});
|
|
if(!response.ok) return;
|
|
streamingCapability=validStreamingCapability(await response.json());
|
|
}catch(_){
|
|
streamingCapability={tts:null,stt:null,preflight:null};
|
|
}
|
|
}
|
|
|
|
function activate(){
|
|
generation+=1;
|
|
const token=generation;
|
|
active=true;
|
|
clearErrorTimer();
|
|
clearSttLanguage();
|
|
modeBtn.classList.add('active');
|
|
toast('Hands-free private voice mode on');
|
|
if(typeof window.stopTTS==='function') window.stopTTS();
|
|
if(typeof S!=='undefined'&&S.busy){setState('thinking');return;}
|
|
startListening(token);
|
|
}
|
|
|
|
function onVoiceClick(event){
|
|
if(!ready) return;
|
|
event.preventDefault();
|
|
event.stopImmediatePropagation();
|
|
if(active) deactivate(true); else activate();
|
|
}
|
|
|
|
async function initialize(){
|
|
try{
|
|
const response=await fetch('/api/transcribe/capability',{cache:'no-store'});
|
|
const capability=await response.json().catch(function(){return {};});
|
|
if(!response.ok||!capability.available||capability.provider!=='local_command') return;
|
|
ready=true;
|
|
discoverStreamingCapability();
|
|
if(localStorage.getItem('hermes-atlas-voice-initialized')!=='1'){
|
|
localStorage.setItem('hermes-atlas-voice-initialized','1');
|
|
localStorage.setItem('hermes-voice-mode-button','true');
|
|
localStorage.setItem('hermes-tts-engine','atlas');
|
|
localStorage.setItem('hermes-tts-enabled','true');
|
|
}
|
|
// Move existing installations to the lower-latency silence window once;
|
|
// subsequent user changes remain untouched.
|
|
if(localStorage.getItem('hermes-atlas-voice-latency-v2')!=='1'){
|
|
localStorage.setItem('hermes-atlas-voice-latency-v2','1');
|
|
localStorage.setItem('hermes-voice-silence-ms','1100');
|
|
}
|
|
const selector=document.getElementById('settingsTtsEngine');
|
|
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
|
const option=document.createElement('option');
|
|
option.value='atlas';
|
|
option.textContent='Atlas Jetson (private)';
|
|
selector.insertBefore(option,selector.firstChild);
|
|
}
|
|
modeBtn.style.display=localStorage.getItem('hermes-voice-mode-button')==='false'?'none':'';
|
|
modeBtn.addEventListener('click',onVoiceClick,true);
|
|
window._applyVoiceModePref=function(){
|
|
if(typeof originalApplyPreference==='function') originalApplyPreference();
|
|
if(ready){
|
|
const enabled=localStorage.getItem('hermes-voice-mode-button')!=='false';
|
|
modeBtn.style.display=enabled?'':'none';
|
|
if(!enabled&&active) deactivate(false);
|
|
}
|
|
};
|
|
window.autoReadLastAssistant=function(){
|
|
if(active){
|
|
if(suppressAutoRead||bargeCancelPromise||state==='listening'||state==='transcribing') return;
|
|
if(state==='thinking'||state==='speaking'){speakResponse(generation);return;}
|
|
}
|
|
if(typeof originalAutoRead==='function') originalAutoRead.apply(this,arguments);
|
|
};
|
|
window._voiceModeActive=function(){return active;};
|
|
window._voiceModeDeactivate=function(){deactivate(false);};
|
|
window._voiceModeImmediateSend=function(){
|
|
if(active&&recorder&&recorder.state!=='inactive') recorder.stop();
|
|
};
|
|
}catch(_){
|
|
// The upstream browser voice implementation remains available as fallback.
|
|
}
|
|
}
|
|
|
|
initialize();
|
|
})();
|