1086 lines
42 KiB
JavaScript
1086 lines
42 KiB
JavaScript
// Natural turn-taking for chat.bstein.dev using the private Jetsons.
|
|
(function(){
|
|
'use strict';
|
|
|
|
const modeBtn=document.getElementById('btnVoiceMode');
|
|
const bar=document.getElementById('voiceModeBar');
|
|
const indicator=document.getElementById('voiceModeIndicator');
|
|
const label=document.getElementById('voiceModeLabel');
|
|
const composer=document.getElementById('msg');
|
|
if(!modeBtn||!bar||!indicator||!label||!composer||!navigator.mediaDevices||!window.MediaRecorder) return;
|
|
|
|
let ready=false;
|
|
let active=false;
|
|
let state='idle';
|
|
let generation=0;
|
|
let turnSequence=0;
|
|
let recorder=null;
|
|
let stream=null;
|
|
let audioContext=null;
|
|
let captureNode=null;
|
|
let vadTimer=null;
|
|
let responsePollTimer=null;
|
|
let currentAudio=null;
|
|
let playbackSession=null;
|
|
let streamingStt=null;
|
|
let captureTurnId='';
|
|
let thinkingTurnId='';
|
|
let thinkingSession=null;
|
|
let assistantBaseline=null;
|
|
let speechTurn=null;
|
|
let errorTimer=null;
|
|
let visualInputLevel=0;
|
|
let streamingCapability={tts:null,stt:null};
|
|
const reducedMotion=window.matchMedia?window.matchMedia('(prefers-reduced-motion: reduce)'):{matches:false};
|
|
const ERROR_VISIBLE_MS=3200;
|
|
const STREAMING_CAPABILITY_URL='/api/voice/streaming/capability';
|
|
const TTS_STREAM_URL='/api/tts/stream';
|
|
const STT_STREAM_PATH='/api/transcribe/stream';
|
|
const WORKLET_URL='static/atlas-voice-worklet.js';
|
|
const STATE_LABELS={
|
|
listening:'Listening',
|
|
transcribing:'Transcribing…',
|
|
thinking:'Thinking…',
|
|
speaking:'Speaking',
|
|
error:'Voice unavailable',
|
|
idle:'',
|
|
};
|
|
// The only language signal this file trusts is the one the private Whisper
|
|
// service returned for the audio of the turn currently being answered. It is
|
|
// bound to that turn's generation token and consumed exactly once.
|
|
let sttLanguage='';
|
|
let sttLanguageToken=-1;
|
|
const TTS_LANGUAGES=['en','ru','es'];
|
|
const originalAutoRead=window.autoReadLastAssistant;
|
|
const originalApplyPreference=window._applyVoiceModePref;
|
|
|
|
function normalizeSttLanguage(value){
|
|
if(typeof value!=='string') return '';
|
|
const code=value.trim().toLowerCase();
|
|
return TTS_LANGUAGES.indexOf(code)>=0?code:'';
|
|
}
|
|
|
|
function clearSttLanguage(){
|
|
sttLanguage='';
|
|
sttLanguageToken=-1;
|
|
}
|
|
|
|
function rememberSttLanguage(language,token){
|
|
sttLanguage=language||'';
|
|
sttLanguageToken=sttLanguage?token:-1;
|
|
}
|
|
|
|
function takeSttLanguage(token){
|
|
const language=sttLanguageToken===token?sttLanguage:'';
|
|
clearSttLanguage();
|
|
return language;
|
|
}
|
|
|
|
function toast(message){
|
|
if(typeof window.showToast==='function') window.showToast(message,3000);
|
|
}
|
|
|
|
function setState(next,customLabel){
|
|
state=next;
|
|
indicator.className='voice-mode-indicator '+next;
|
|
bar.dataset.voiceState=next;
|
|
bar.setAttribute('aria-busy',next==='transcribing'||next==='thinking'?'true':'false');
|
|
label.textContent=customLabel||STATE_LABELS[next]||'';
|
|
bar.style.display=(active&&next!=='idle')||next==='error'?'':'none';
|
|
resetInputLevel();
|
|
}
|
|
|
|
function resetInputLevel(){
|
|
visualInputLevel=0;
|
|
indicator.style.setProperty('--voice-ripple-scale','1.035');
|
|
indicator.style.setProperty('--voice-ripple-opacity','0.3');
|
|
}
|
|
|
|
function updateInputLevel(rms){
|
|
if(reducedMotion.matches||state!=='listening') return;
|
|
const target=Math.max(0,Math.min(1,(rms-0.01)/0.18));
|
|
visualInputLevel=(visualInputLevel*0.72)+(target*0.28);
|
|
indicator.style.setProperty('--voice-ripple-scale',(1.035+(visualInputLevel*0.16)).toFixed(3));
|
|
indicator.style.setProperty('--voice-ripple-opacity',(0.26+(visualInputLevel*0.48)).toFixed(3));
|
|
}
|
|
|
|
function clearErrorTimer(){
|
|
if(errorTimer){window.clearTimeout(errorTimer);errorTimer=null;}
|
|
}
|
|
|
|
function errorMessage(error,fallback){
|
|
return String((error&&error.message)||fallback||'Voice unavailable').trim();
|
|
}
|
|
|
|
function cancelSpeechTurn(){
|
|
if(!speechTurn) return;
|
|
speechTurn.cancelled=true;
|
|
speechTurn.final=true;
|
|
while(speechTurn.waiters.length) speechTurn.waiters.shift()(null);
|
|
speechTurn=null;
|
|
}
|
|
|
|
function cancelStreamingStt(){
|
|
if(!streamingStt) return;
|
|
streamingStt.cancel();
|
|
streamingStt=null;
|
|
}
|
|
|
|
function showUnavailable(message){
|
|
generation+=1;
|
|
active=false;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
stopResponseObserver();
|
|
cancelSpeechTurn();
|
|
stopCapture();
|
|
stopPlayback();
|
|
modeBtn.classList.remove('active');
|
|
setState('error',message);
|
|
clearErrorTimer();
|
|
errorTimer=window.setTimeout(function(){
|
|
errorTimer=null;
|
|
if(!active&&state==='error'&&indicator.classList.contains('error')) setState('idle');
|
|
},ERROR_VISIBLE_MS);
|
|
}
|
|
|
|
function stopCapture(){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
if(recorder&&recorder.state!=='inactive'){
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
recorder=null;
|
|
captureNode=null;
|
|
cancelStreamingStt();
|
|
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
|
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
|
}
|
|
|
|
function stopPlayback(){
|
|
if(currentAudio){
|
|
try{currentAudio.pause();currentAudio.currentTime=0;}catch(_){ }
|
|
currentAudio=null;
|
|
}
|
|
if(playbackSession){
|
|
playbackSession.cancelled=true;
|
|
if(playbackSession.blobWake){playbackSession.blobWake();playbackSession.blobWake=null;}
|
|
playbackSession.controllers.forEach(function(controller){controller.abort();});
|
|
if(playbackSession.lowWaterWake){
|
|
playbackSession.lowWaterWake();
|
|
playbackSession.lowWaterWake=null;
|
|
}
|
|
if(playbackSession.drainWake){playbackSession.drainWake();playbackSession.drainWake=null;}
|
|
if(playbackSession.node){
|
|
try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ }
|
|
}
|
|
if(playbackSession.context){try{playbackSession.context.close();}catch(_){ }}
|
|
playbackSession=null;
|
|
}
|
|
indicator.classList.remove('is-playing');
|
|
}
|
|
|
|
function stopResponseObserver(){
|
|
if(responsePollTimer){clearInterval(responsePollTimer);responsePollTimer=null;}
|
|
}
|
|
|
|
function deactivate(showMessage){
|
|
generation+=1;
|
|
active=false;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
clearErrorTimer();
|
|
clearSttLanguage();
|
|
stopResponseObserver();
|
|
cancelSpeechTurn();
|
|
stopCapture();
|
|
stopPlayback();
|
|
modeBtn.classList.remove('active');
|
|
setState('idle');
|
|
if(showMessage) toast('Hands-free voice mode off');
|
|
}
|
|
|
|
function restartSoon(token,delay){
|
|
window.setTimeout(function(){
|
|
if(active&&token===generation) startListening(token);
|
|
},delay||500);
|
|
}
|
|
|
|
function assistantRows(){
|
|
return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
|
|
}
|
|
|
|
function readAssistantRow(row){
|
|
if(!row) return '';
|
|
if(row.dataset&&typeof row.dataset.rawText==='string') return row.dataset.rawText;
|
|
return typeof row.textContent==='string'?row.textContent:'';
|
|
}
|
|
|
|
function rememberAssistantBaseline(){
|
|
const rows=assistantRows();
|
|
const row=rows.length?rows[rows.length-1]:null;
|
|
assistantBaseline={row:row,text:readAssistantRow(row),count:rows.length};
|
|
}
|
|
|
|
function sendTranscript(transcript,token,language){
|
|
if(!active||token!==generation) return;
|
|
const text=String(transcript||'').trim();
|
|
if(!text){clearSttLanguage();restartSoon(token,350);return;}
|
|
composer.value=text;
|
|
if(typeof window.autoResize==='function') window.autoResize();
|
|
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
thinkingTurnId=captureTurnId||String(token)+'-'+String(++turnSequence);
|
|
rememberAssistantBaseline();
|
|
rememberSttLanguage(language,token);
|
|
setState('thinking');
|
|
startResponseObserver(token);
|
|
if(typeof window.send==='function') window.send();
|
|
}
|
|
|
|
function audioExtension(mimeType){
|
|
const normalized=String(mimeType||'').toLowerCase();
|
|
if(normalized.indexOf('ogg')>=0) return 'ogg';
|
|
if(normalized.indexOf('mp4')>=0) return 'mp4';
|
|
return 'webm';
|
|
}
|
|
|
|
async function transcribe(blob,token){
|
|
if(!active||token!==generation) return;
|
|
setState('transcribing');
|
|
const ext=audioExtension(blob.type);
|
|
const form=new FormData();
|
|
form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
|
|
try{
|
|
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
|
const payload=await response.json().catch(function(){return {};});
|
|
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
|
sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language));
|
|
}catch(error){
|
|
if(!active||token!==generation) return;
|
|
const message=errorMessage(error,'Private Whisper is unavailable');
|
|
showUnavailable(message);
|
|
toast(message);
|
|
// If the browser supplies its own recognizer, hand control back to the
|
|
// upstream voice implementation until the Jetson becomes healthy again.
|
|
if(window.SpeechRecognition||window.webkitSpeechRecognition){
|
|
modeBtn.removeEventListener('click',onVoiceClick,true);
|
|
window.setTimeout(function(){modeBtn.click();},50);
|
|
}
|
|
}
|
|
}
|
|
|
|
function websocketUrl(path){
|
|
const protocol=window.location&&window.location.protocol==='https:'?'wss:':'ws:';
|
|
return protocol+'//'+window.location.host+path;
|
|
}
|
|
|
|
function encodePcm16(samples){
|
|
const bytes=new ArrayBuffer(samples.length*2);
|
|
const view=new DataView(bytes);
|
|
for(let index=0;index<samples.length;index+=1){
|
|
const value=Math.max(-1,Math.min(1,samples[index]));
|
|
view.setInt16(index*2,value<0?value*32768:value*32767,true);
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
function createResampler(sourceRate,targetRate){
|
|
let tail=null;
|
|
let position=0;
|
|
const ratio=sourceRate/targetRate;
|
|
return function(samples){
|
|
const input=new Float32Array(samples.length+(tail===null?0:1));
|
|
let offset=0;
|
|
if(tail!==null){input[0]=tail;offset=1;}
|
|
input.set(samples,offset);
|
|
if(input.length<2){tail=input.length?input[0]:tail;return new Float32Array(0);}
|
|
const output=[];
|
|
while(position<input.length-1){
|
|
const left=Math.floor(position);
|
|
const fraction=position-left;
|
|
output.push(input[left]+((input[left+1]-input[left])*fraction));
|
|
position+=ratio;
|
|
}
|
|
position-=input.length-1;
|
|
tail=input[input.length-1];
|
|
return new Float32Array(output);
|
|
};
|
|
}
|
|
|
|
function createStreamingSttSession(turnId,context){
|
|
if(!streamingCapability.stt||!window.WebSocket||!window.location) return null;
|
|
let socket;
|
|
let settled=false;
|
|
let committed=false;
|
|
let speculative=false;
|
|
let workletReady=false;
|
|
let queue=[];
|
|
let resolveFinal;
|
|
let rejectFinal;
|
|
const finalPromise=new Promise(function(resolve,reject){resolveFinal=resolve;rejectFinal=reject;});
|
|
// A transport can fail before recorder.onstop awaits it. Keep the rejection
|
|
// observed without changing what the later await receives.
|
|
finalPromise.catch(function(){ });
|
|
const resample=createResampler(context.sampleRate,16000);
|
|
|
|
function reject(error){
|
|
if(settled) return;
|
|
settled=true;
|
|
rejectFinal(error instanceof Error?error:new Error(String(error||'Streaming transcription failed')));
|
|
}
|
|
function sendJson(payload){
|
|
if(socket&&socket.readyState===1){socket.send(JSON.stringify(payload));return true;}
|
|
return false;
|
|
}
|
|
function flush(){
|
|
if(!socket||socket.readyState!==1) return;
|
|
while(queue.length&&socket.bufferedAmount<524288) socket.send(queue.shift());
|
|
if(queue.length) window.setTimeout(flush,20);
|
|
}
|
|
try{
|
|
const csrf=String((window.__HERMES_CONFIG__&&window.__HERMES_CONFIG__.csrfToken)||'');
|
|
const protocols=['hermes-voice-v1'];
|
|
if(csrf) protocols.push('hermes-csrf.'+csrf);
|
|
socket=new WebSocket(websocketUrl(STT_STREAM_PATH),protocols);
|
|
socket.binaryType='arraybuffer';
|
|
}catch(error){
|
|
reject(error);
|
|
return null;
|
|
}
|
|
socket.onopen=function(){
|
|
sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:'auto'});
|
|
flush();
|
|
};
|
|
socket.onmessage=function(event){
|
|
if(typeof event.data!=='string') return;
|
|
let payload;
|
|
try{payload=JSON.parse(event.data);}catch(_){return;}
|
|
if(payload.turn_id!==turnId) return;
|
|
if(payload.type==='final'&&!settled){
|
|
settled=true;
|
|
resolveFinal({transcript:payload.transcript||'',language:normalizeSttLanguage(payload.language)});
|
|
}else if(payload.type==='error'){
|
|
reject(new Error(payload.error||'Streaming transcription failed'));
|
|
}
|
|
};
|
|
socket.onerror=function(){reject(new Error('Streaming transcription connection failed'));};
|
|
socket.onclose=function(){if(!settled) reject(new Error('Streaming transcription closed before final result'));};
|
|
|
|
return {
|
|
finalPromise:finalPromise,
|
|
setWorkletReady:function(value){workletReady=value;},
|
|
push:function(samples){
|
|
if(settled||committed||!workletReady) return;
|
|
const pcm=resample(samples);
|
|
if(!pcm.length) return;
|
|
queue.push(encodePcm16(pcm));
|
|
flush();
|
|
},
|
|
speculate:function(){
|
|
if(settled||committed||speculative) return;
|
|
speculative=true;
|
|
sendJson({type:'speculate',turn_id:turnId});
|
|
},
|
|
resume:function(){
|
|
if(settled||committed||!speculative) return;
|
|
speculative=false;
|
|
sendJson({type:'resume',turn_id:turnId});
|
|
},
|
|
commit:function(){
|
|
if(settled||committed) return finalPromise;
|
|
committed=true;
|
|
flush();
|
|
const commitDeadline=Date.now()+30000;
|
|
const sendCommit=function(){
|
|
if(settled) return;
|
|
if(!socket||socket.readyState>1||Date.now()>=commitDeadline){
|
|
reject(new Error('Streaming transcription closed before commit'));
|
|
return;
|
|
}
|
|
if(socket.readyState!==1||queue.length){flush();window.setTimeout(sendCommit,20);return;}
|
|
if(!sendJson({type:'commit',turn_id:turnId})) reject(new Error('Streaming transcription commit failed'));
|
|
};
|
|
sendCommit();
|
|
return finalPromise;
|
|
},
|
|
cancel:function(){
|
|
if(!settled){sendJson({type:'cancel',turn_id:turnId});reject(new Error('Streaming transcription cancelled'));}
|
|
if(socket&&socket.readyState<2) socket.close(1000,'cancelled');
|
|
},
|
|
};
|
|
}
|
|
|
|
async function installCaptureWorklet(context,source,session){
|
|
if(!session) return false;
|
|
if(!context.audioWorklet||!window.AudioWorkletNode){
|
|
session.cancel();
|
|
if(streamingStt===session) streamingStt=null;
|
|
return false;
|
|
}
|
|
try{
|
|
await context.audioWorklet.addModule(WORKLET_URL);
|
|
if(!streamingStt||streamingStt!==session) return false;
|
|
const node=new AudioWorkletNode(context,'atlas-pcm-capture');
|
|
const silentGain=context.createGain();
|
|
silentGain.gain.value=0;
|
|
let flushResolve=null;
|
|
node.port.onmessage=function(event){
|
|
if(!event.data) return;
|
|
if(event.data.type==='flushed'&&flushResolve){flushResolve();flushResolve=null;return;}
|
|
if(!streamingStt||streamingStt!==session||event.data.type!=='pcm') return;
|
|
session.push(new Float32Array(event.data.samples));
|
|
};
|
|
node._atlasFlush=function(){
|
|
return new Promise(function(resolve){flushResolve=resolve;node.port.postMessage({type:'flush'});});
|
|
};
|
|
source.connect(node);
|
|
node.connect(silentGain);
|
|
silentGain.connect(context.destination);
|
|
captureNode=node;
|
|
session.setWorkletReady(true);
|
|
return true;
|
|
}catch(_){
|
|
session.cancel();
|
|
if(streamingStt===session) streamingStt=null;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function transcribeStreamingOrFallback(blob,token,session){
|
|
if(!active||token!==generation) return;
|
|
setState('transcribing');
|
|
if(session){
|
|
try{
|
|
const result=await session.commit();
|
|
if(!String(result.transcript||'').trim()) throw new Error('Streaming transcription returned no final text');
|
|
if(active&&token===generation){
|
|
streamingStt=null;
|
|
sendTranscript(result.transcript,token,result.language);
|
|
return;
|
|
}
|
|
}catch(_){
|
|
// The finalized browser container below is the quality-preserving path
|
|
// whenever rolling PCM transport or speculative inference fails.
|
|
}
|
|
if(streamingStt===session) streamingStt=null;
|
|
}
|
|
transcribe(blob,token);
|
|
}
|
|
|
|
async function startListening(token){
|
|
if(!active||token!==generation) return;
|
|
stopCapture();
|
|
stopPlayback();
|
|
cancelSpeechTurn();
|
|
clearSttLanguage();
|
|
captureTurnId=String(token)+'-'+String(++turnSequence);
|
|
setState('listening');
|
|
try{
|
|
const capture=await navigator.mediaDevices.getUserMedia({
|
|
audio:(function(){
|
|
const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
|
|
const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
|
|
if(supported.voiceIsolation) constraints.voiceIsolation=true;
|
|
return constraints;
|
|
})(),
|
|
});
|
|
if(!active||token!==generation){capture.getTracks().forEach(function(track){track.stop();});return;}
|
|
stream=capture;
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
try{
|
|
// Let the browser's native resampler produce Whisper's 16 kHz input.
|
|
audioContext=new Context({sampleRate:16000,latencyHint:'interactive'});
|
|
}catch(_){
|
|
audioContext=new Context();
|
|
}
|
|
const analyser=audioContext.createAnalyser();
|
|
analyser.fftSize=2048;
|
|
const highpass=audioContext.createBiquadFilter();
|
|
highpass.type='highpass';
|
|
highpass.frequency.value=140;
|
|
highpass.Q.value=0.7;
|
|
const mediaSource=audioContext.createMediaStreamSource(stream);
|
|
mediaSource.connect(highpass);
|
|
highpass.connect(analyser);
|
|
streamingStt=createStreamingSttSession(captureTurnId,audioContext);
|
|
// Whisper receives the browser's full-band processed microphone signal;
|
|
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
|
|
// needlessly altered before final recognition.
|
|
if(streamingStt) await installCaptureWorklet(audioContext,mediaSource,streamingStt);
|
|
const samples=new Uint8Array(analyser.fftSize);
|
|
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm'];
|
|
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
|
const chunks=[];
|
|
let heardSpeech=false;
|
|
let voiceFrames=0;
|
|
let noiseFloor=0.008;
|
|
let lastSpeech=Date.now();
|
|
let speculative=false;
|
|
const started=Date.now();
|
|
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
|
let recordedMime=recorder.mimeType||mime||'';
|
|
recorder.ondataavailable=function(event){
|
|
if(!event.data||!event.data.size) return;
|
|
if(event.data.type) recordedMime=event.data.type;
|
|
chunks.push(event.data);
|
|
};
|
|
recorder.onstop=async function(){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
const finalCaptureNode=captureNode;
|
|
if(finalCaptureNode&&typeof finalCaptureNode._atlasFlush==='function'){
|
|
await Promise.race([
|
|
finalCaptureNode._atlasFlush(),
|
|
new Promise(function(resolve){window.setTimeout(resolve,100);}),
|
|
]);
|
|
}
|
|
const recordedStream=stream;
|
|
stream=null;
|
|
if(recordedStream) recordedStream.getTracks().forEach(function(track){track.stop();});
|
|
const context=audioContext;
|
|
audioContext=null;
|
|
if(context){try{context.close();}catch(_){ }}
|
|
captureNode=null;
|
|
recorder=null;
|
|
if(!active||token!==generation){cancelStreamingStt();return;}
|
|
if(!heardSpeech||!chunks.length){cancelStreamingStt();restartSoon(token,300);return;}
|
|
const session=streamingStt;
|
|
transcribeStreamingOrFallback(new Blob(chunks,{type:recordedMime||'audio/webm'}),token,session);
|
|
};
|
|
// Ask the browser for one finalized container at stop. Android Chromium
|
|
// can emit timeslice fragments without a reusable EBML initialization
|
|
// header; concatenating those fragments made otherwise valid recordings
|
|
// intermittently unreadable by ffmpeg. A bounded 90-second Opus capture
|
|
// is small enough to retain as one browser-owned recording.
|
|
recorder.start();
|
|
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1100',10)||1100);
|
|
const speculateMs=Math.min(silenceMs-250,Math.max(450,Math.round(silenceMs*0.55)));
|
|
vadTimer=window.setInterval(function(){
|
|
if(!active||token!==generation||!recorder||recorder.state==='inactive') return;
|
|
analyser.getByteTimeDomainData(samples);
|
|
let energy=0;
|
|
for(let index=0;index<samples.length;index+=1){
|
|
const normalized=(samples[index]-128)/128;
|
|
energy+=normalized*normalized;
|
|
}
|
|
const rms=Math.sqrt(energy/samples.length);
|
|
updateInputLevel(rms);
|
|
const now=Date.now();
|
|
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
|
const voiceNow=rms>speechThreshold;
|
|
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
|
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
|
if(!heardSpeech&&voiceFrames>=3){
|
|
heardSpeech=true;
|
|
lastSpeech=now;
|
|
}else if(heardSpeech&&voiceNow){
|
|
lastSpeech=now;
|
|
if(speculative&&streamingStt){streamingStt.resume();speculative=false;}
|
|
}
|
|
if(heardSpeech&&!voiceNow&&!speculative&&(now-lastSpeech)>=speculateMs&&streamingStt){
|
|
streamingStt.speculate();
|
|
speculative=true;
|
|
}
|
|
const finished=heardSpeech&&(now-lastSpeech)>=silenceMs;
|
|
const timedOut=now-started>=90000;
|
|
const idle=(!heardSpeech)&&(now-started)>=20000;
|
|
if(finished||timedOut||idle){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
},100);
|
|
}catch(error){
|
|
if(!active||token!==generation) return;
|
|
const message=errorMessage(error,'Microphone permission is required');
|
|
showUnavailable(message);
|
|
toast(message);
|
|
}
|
|
}
|
|
|
|
function cleanForSpeech(text){
|
|
if(typeof window._stripForTTS==='function') return window._stripForTTS(text);
|
|
return String(text||'').replace(/```[\s\S]*?```/g,' code block ').replace(/\s+/g,' ').trim();
|
|
}
|
|
|
|
function sentenceEnd(text){
|
|
const source=String(text||'');
|
|
const boundary=/[.!?…]+["')\]}]*(?:\s|$)/g;
|
|
const abbreviation=/(?:\b(?:mr|mrs|ms|dr|prof|sr|jr|st|vs|etc|e\.g|i\.e)|\b[A-Z])\.$/i;
|
|
let match;
|
|
while((match=boundary.exec(source))){
|
|
const end=match.index+match[0].trimEnd().length;
|
|
const prefix=source.slice(0,end).replace(/["')\]}]+$/,'');
|
|
if(prefix.endsWith('.')&&abbreviation.test(prefix)) continue;
|
|
return end;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
function preferredCut(text,min,target,max){
|
|
const bounded=text.slice(0,max);
|
|
const candidates=[];
|
|
const punctuation=/[,;:—–.!?…]+["')\]}]*(?:\s|$)/g;
|
|
let match;
|
|
while((match=punctuation.exec(bounded))){
|
|
const end=match.index+match[0].trimEnd().length;
|
|
if(end>=min) candidates.push({end:end,weight:Math.abs(end-target)});
|
|
}
|
|
if(candidates.length){
|
|
candidates.sort(function(left,right){return left.weight-right.weight;});
|
|
return candidates[0].end;
|
|
}
|
|
const spaces=[];
|
|
const whitespace=/\s+/g;
|
|
while((match=whitespace.exec(bounded))){if(match.index>=min) spaces.push(match.index);}
|
|
if(spaces.length){
|
|
spaces.sort(function(left,right){return Math.abs(left-target)-Math.abs(right-target);});
|
|
return spaces[0];
|
|
}
|
|
return Math.min(max,text.length);
|
|
}
|
|
|
|
function adaptiveChunks(text,final,firstChunk){
|
|
const source=String(text||'').trim();
|
|
if(!source) return {chunks:[],consumed:0};
|
|
const chunks=[];
|
|
let offset=0;
|
|
if(firstChunk!==false){
|
|
const firstSentence=sentenceEnd(source);
|
|
if(firstSentence<0&&!final) return {chunks:[],consumed:0};
|
|
const firstAvailable=firstSentence>=0?firstSentence:source.length;
|
|
const firstSlice=source.slice(0,firstAvailable);
|
|
const firstCut=firstSlice.length<=60?firstSlice.length:preferredCut(firstSlice,40,52,60);
|
|
chunks.push(source.slice(0,firstCut).trim());
|
|
offset=firstCut;
|
|
}
|
|
while(offset<source.length){
|
|
const remaining=source.slice(offset).trimStart();
|
|
offset=source.length-remaining.length;
|
|
if(!remaining) break;
|
|
if(remaining.length<100){
|
|
const complete=sentenceEnd(remaining);
|
|
if(!final&&complete<0) break;
|
|
const end=final?remaining.length:complete;
|
|
chunks.push(remaining.slice(0,end).trim());
|
|
offset+=end;
|
|
continue;
|
|
}
|
|
const cut=preferredCut(remaining,100,120,140);
|
|
if(!final&&cut===remaining.length&&remaining.length<140) break;
|
|
chunks.push(remaining.slice(0,cut).trim());
|
|
offset+=cut;
|
|
}
|
|
return {chunks:chunks.filter(Boolean),consumed:offset};
|
|
}
|
|
|
|
// Export the deterministic splitter as a narrow diagnostic/test seam.
|
|
window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;};
|
|
|
|
function playBlob(blob,token){
|
|
return new Promise(function(resolve,reject){
|
|
if(!active||token!==generation){resolve();return;}
|
|
const session=playbackSession;
|
|
const url=URL.createObjectURL(blob);
|
|
const audio=new Audio(url);
|
|
currentAudio=audio;
|
|
let settled=false;
|
|
function cleanup(callback,value){
|
|
if(settled) return;
|
|
settled=true;
|
|
if(currentAudio===audio) currentAudio=null;
|
|
if(session&&session.blobWake===cancel) session.blobWake=null;
|
|
indicator.classList.remove('is-playing');
|
|
URL.revokeObjectURL(url);
|
|
callback(value);
|
|
}
|
|
function cancel(){cleanup(resolve);}
|
|
if(session) session.blobWake=cancel;
|
|
audio.onended=function(){cleanup(resolve);};
|
|
audio.onerror=function(){cleanup(reject,new Error('Local speech playback failed'));};
|
|
audio.play().then(function(){
|
|
if(active&&token===generation&¤tAudio===audio) indicator.classList.add('is-playing');
|
|
}).catch(function(error){cleanup(reject,error);});
|
|
});
|
|
}
|
|
|
|
function ttsRequest(chunk,language,turnId){
|
|
const request={text:chunk,engine:'atlas',turn_id:turnId};
|
|
if(language) request.language=language;
|
|
return request;
|
|
}
|
|
|
|
async function fetchSpeech(chunk,language,turnId){
|
|
// `language` is only ever the private STT result for this turn. When it is
|
|
// absent the field is omitted entirely and the server picks English.
|
|
const response=await fetch('/api/tts',{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json'},
|
|
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
|
});
|
|
if(!response.ok){
|
|
const payload=await response.json().catch(function(){return {};});
|
|
throw new Error(payload.error||('Local speech request failed: '+response.status));
|
|
}
|
|
return response.blob();
|
|
}
|
|
|
|
async function prepareSpeech(chunk,language,turnId,token){
|
|
if(streamingCapability.tts&&window.ReadableStream&&window.AudioWorkletNode){
|
|
const controller=new AbortController();
|
|
if(playbackSession) playbackSession.controllers.add(controller);
|
|
try{
|
|
const response=await fetch(TTS_STREAM_URL,{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json','Accept':'audio/pcm'},
|
|
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
|
signal:controller.signal,
|
|
});
|
|
if(!response.ok||!response.body) throw new Error('Streaming speech unavailable');
|
|
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||String(streamingCapability.tts.sample_rate||22050),10);
|
|
const channels=parseInt(response.headers.get('X-Audio-Channels')||'1',10);
|
|
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000||channels!==1) throw new Error('Unsupported streaming speech format');
|
|
return {kind:'pcm',response:response,sampleRate:sampleRate,controller:controller,token:token,chunk:chunk,language:language,turnId:turnId,started:false};
|
|
}catch(error){
|
|
if(playbackSession) playbackSession.controllers.delete(controller);
|
|
if(error&&error.name==='AbortError') throw error;
|
|
// Disable only this optional transport for the browser session. The
|
|
// complete WAV endpoint preserves voice quality and availability.
|
|
streamingCapability.tts=null;
|
|
}
|
|
}
|
|
return {kind:'blob',blob:await fetchSpeech(chunk,language,turnId)};
|
|
}
|
|
|
|
async function playPcm(asset,token){
|
|
if(!active||token!==generation) return;
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
let context;
|
|
try{
|
|
// Keep Piper at its native rate and let the browser/audio device own any
|
|
// final hardware conversion; the worklet interpolator is a fallback.
|
|
context=new Context({sampleRate:asset.sampleRate,latencyHint:'interactive'});
|
|
}catch(_){
|
|
context=new Context();
|
|
}
|
|
await context.audioWorklet.addModule(WORKLET_URL);
|
|
if(!active||token!==generation){context.close();return;}
|
|
const node=new AudioWorkletNode(context,'atlas-pcm-playback');
|
|
const session=playbackSession;
|
|
if(!session||session.cancelled){context.close();return;}
|
|
session.context=context;
|
|
session.node=node;
|
|
node.connect(context.destination);
|
|
await context.resume();
|
|
let bufferedFrames=0;
|
|
let lowWaterResolve=null;
|
|
let ended=false;
|
|
const drained=new Promise(function(resolve,reject){
|
|
session.drainWake=resolve;
|
|
node.port.onmessage=function(event){
|
|
const data=event.data||{};
|
|
if(data.type==='buffer'){
|
|
bufferedFrames=data.frames||0;
|
|
if(bufferedFrames<asset.sampleRate&&lowWaterResolve){lowWaterResolve();lowWaterResolve=null;}
|
|
}else if(data.type==='drained'){
|
|
ended=true;
|
|
session.drainWake=null;
|
|
resolve();
|
|
}else if(data.type==='error') reject(new Error(data.error||'Streaming speech playback failed'));
|
|
};
|
|
});
|
|
indicator.classList.add('is-playing');
|
|
const reader=asset.response.body.getReader();
|
|
let carry=null;
|
|
try{
|
|
while(active&&token===generation&&!session.cancelled){
|
|
if(bufferedFrames>asset.sampleRate*2){
|
|
await new Promise(function(resolve){
|
|
let completed=false;
|
|
const wake=function(){if(completed)return;completed=true;lowWaterResolve=null;session.lowWaterWake=null;resolve();};
|
|
lowWaterResolve=wake;
|
|
session.lowWaterWake=wake;
|
|
window.setTimeout(wake,3000);
|
|
});
|
|
}
|
|
const result=await reader.read();
|
|
if(result.done) break;
|
|
let bytes=result.value;
|
|
if(carry!==null){
|
|
const joined=new Uint8Array(bytes.length+1);
|
|
joined[0]=carry;joined.set(bytes,1);bytes=joined;carry=null;
|
|
}
|
|
if(bytes.length%2){carry=bytes[bytes.length-1];bytes=bytes.slice(0,-1);}
|
|
if(!bytes.length) continue;
|
|
const copy=bytes.buffer.slice(bytes.byteOffset,bytes.byteOffset+bytes.byteLength);
|
|
bufferedFrames+=bytes.byteLength/2;
|
|
asset.started=true;
|
|
node.port.postMessage({type:'push',samples:copy,sampleRate:asset.sampleRate},[copy]);
|
|
}
|
|
node.port.postMessage({type:'end'});
|
|
if(active&&token===generation&&!session.cancelled) await drained;
|
|
}finally{
|
|
if(!ended){try{await reader.cancel();}catch(_){ }}
|
|
indicator.classList.remove('is-playing');
|
|
try{node.disconnect();context.close();}catch(_){ }
|
|
session.node=null;
|
|
session.context=null;
|
|
session.drainWake=null;
|
|
session.controllers.delete(asset.controller);
|
|
}
|
|
}
|
|
|
|
async function playPrepared(asset,token){
|
|
if(asset.kind!=='pcm') return playBlob(asset.blob,token);
|
|
try{
|
|
return await playPcm(asset,token);
|
|
}catch(error){
|
|
if(asset.started) throw error;
|
|
streamingCapability.tts=null;
|
|
return playBlob(await fetchSpeech(asset.chunk,asset.language,asset.turnId),token);
|
|
}
|
|
}
|
|
|
|
function nextSpeechChunk(turn){
|
|
if(turn.cancelled) return Promise.resolve(null);
|
|
if(turn.queue.length) return Promise.resolve(turn.queue.shift());
|
|
if(turn.final) return Promise.resolve(null);
|
|
return new Promise(function(resolve){turn.waiters.push(resolve);});
|
|
}
|
|
|
|
function enqueueSpeech(turn,chunks){
|
|
chunks.forEach(function(chunk){
|
|
if(!chunk) return;
|
|
if(turn.waiters.length) turn.waiters.shift()(chunk); else turn.queue.push(chunk);
|
|
});
|
|
}
|
|
|
|
function finishSpeechQueue(turn){
|
|
turn.final=true;
|
|
while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null);
|
|
}
|
|
|
|
async function runSpeechQueue(turn){
|
|
if(turn.running) return;
|
|
turn.running=true;
|
|
playbackSession={turnId:turn.turnId,controllers:new Set(),context:null,node:null,cancelled:false,lowWaterWake:null,drainWake:null,blobWake:null};
|
|
const session=playbackSession;
|
|
try{
|
|
let chunk=await nextSpeechChunk(turn);
|
|
let current=chunk?prepareSpeech(chunk,turn.language,turn.turnId,turn.token):null;
|
|
while(current){
|
|
const asset=await current;
|
|
// At most one later synthesis request exists while this asset plays.
|
|
// If streaming has not produced it yet, the waiter starts it the
|
|
// instant a punctuation-safe chunk arrives.
|
|
const next=nextSpeechChunk(turn);
|
|
const nextPrepared=next.then(function(value){
|
|
return value?prepareSpeech(value,turn.language,turn.turnId,turn.token):null;
|
|
});
|
|
await playPrepared(asset,turn.token);
|
|
if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return;
|
|
current=await nextPrepared;
|
|
}
|
|
if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300);
|
|
}catch(error){
|
|
if(active&&turn.token===generation&&!turn.cancelled){
|
|
const message=errorMessage(error,'Local speech is unavailable');
|
|
setState('error',message);
|
|
toast(message);
|
|
restartSoon(turn.token,500);
|
|
}
|
|
}finally{
|
|
if(playbackSession===session) stopPlayback();
|
|
if(speechTurn===turn) speechTurn=null;
|
|
}
|
|
}
|
|
|
|
function currentAssistantText(){
|
|
const rows=assistantRows();
|
|
if(!rows.length) return '';
|
|
const row=rows[rows.length-1];
|
|
const text=readAssistantRow(row);
|
|
if(assistantBaseline&&row===assistantBaseline.row&&text===assistantBaseline.text) return '';
|
|
if(assistantBaseline&&rows.length<assistantBaseline.count) return '';
|
|
return cleanForSpeech(text);
|
|
}
|
|
|
|
function ensureSpeechTurn(token){
|
|
if(speechTurn) return speechTurn;
|
|
speechTurn={
|
|
token:token,
|
|
turnId:thinkingTurnId||String(token)+'-'+String(++turnSequence),
|
|
language:takeSttLanguage(token),
|
|
sourceText:'',
|
|
consumed:0,
|
|
first:true,
|
|
queue:[],
|
|
waiters:[],
|
|
final:false,
|
|
cancelled:false,
|
|
running:false,
|
|
};
|
|
return speechTurn;
|
|
}
|
|
|
|
function pumpAssistantResponse(token,isFinal){
|
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
|
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
clearSttLanguage();
|
|
stopResponseObserver();
|
|
cancelSpeechTurn();
|
|
stopPlayback();
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
const text=currentAssistantText();
|
|
if(!text){
|
|
if(isFinal){thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();restartSoon(token,250);}
|
|
return;
|
|
}
|
|
const turn=ensureSpeechTurn(token);
|
|
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
|
|
// Renderers can revise the still-unspoken tail. Already-spoken text is
|
|
// immutable. On the completion callback, advance from the old spoken
|
|
// offset and close the queue even when the renderer rewrote an earlier
|
|
// span; replaying the revised prefix would be more disruptive than
|
|
// preserving the already-heard words.
|
|
if(turn.consumed>0){
|
|
if(!isFinal) return;
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
stopResponseObserver();
|
|
const revisedTail=text.slice(Math.min(turn.consumed,text.length)).trim();
|
|
if(revisedTail){
|
|
const revised=adaptiveChunks(revisedTail,true,false);
|
|
enqueueSpeech(turn,revised.chunks.length?revised.chunks:[revisedTail]);
|
|
turn.consumed=text.length;
|
|
}
|
|
finishSpeechQueue(turn);
|
|
return;
|
|
}
|
|
turn.sourceText='';
|
|
}
|
|
turn.sourceText=text;
|
|
const remaining=text.slice(turn.consumed).trimStart();
|
|
const skipped=text.slice(turn.consumed).length-remaining.length;
|
|
const extracted=adaptiveChunks(remaining,!!isFinal,turn.first);
|
|
if(extracted.chunks.length){
|
|
turn.first=false;
|
|
turn.consumed+=skipped+extracted.consumed;
|
|
enqueueSpeech(turn,extracted.chunks);
|
|
setState('speaking');
|
|
runSpeechQueue(turn);
|
|
}
|
|
if(isFinal){
|
|
thinkingSession=null;
|
|
thinkingTurnId='';
|
|
stopResponseObserver();
|
|
const tail=text.slice(turn.consumed).trim();
|
|
if(tail){enqueueSpeech(turn,[tail]);turn.consumed=text.length;}
|
|
finishSpeechQueue(turn);
|
|
}
|
|
}
|
|
|
|
function startResponseObserver(token){
|
|
stopResponseObserver();
|
|
// A bounded poll is deliberately used in addition to the upstream
|
|
// completion callback. It sees data-raw-text while SSE is still appending,
|
|
// so the first complete sentence can reach Piper before generation ends.
|
|
responsePollTimer=window.setInterval(function(){pumpAssistantResponse(token,false);},75);
|
|
}
|
|
|
|
function speakResponse(token){
|
|
pumpAssistantResponse(token,true);
|
|
}
|
|
|
|
function validStreamingCapability(payload){
|
|
if(!payload||typeof payload!=='object') return {tts:null,stt:null};
|
|
const tts=payload.tts&&payload.tts.available===true&&payload.tts.transport==='http'&&payload.tts.format==='pcm_s16le'?payload.tts:null;
|
|
const stt=payload.stt&&payload.stt.available===true&&payload.stt.transport==='websocket'&&payload.stt.format==='pcm_s16le'&&payload.stt.sample_rate===16000?payload.stt:null;
|
|
return {tts:tts,stt:stt};
|
|
}
|
|
|
|
async function discoverStreamingCapability(){
|
|
try{
|
|
const response=await fetch(STREAMING_CAPABILITY_URL,{cache:'no-store'});
|
|
if(!response.ok) return;
|
|
streamingCapability=validStreamingCapability(await response.json());
|
|
}catch(_){
|
|
streamingCapability={tts:null,stt:null};
|
|
}
|
|
}
|
|
|
|
function activate(){
|
|
generation+=1;
|
|
const token=generation;
|
|
active=true;
|
|
clearErrorTimer();
|
|
clearSttLanguage();
|
|
modeBtn.classList.add('active');
|
|
toast('Hands-free private voice mode on');
|
|
if(typeof window.stopTTS==='function') window.stopTTS();
|
|
if(typeof S!=='undefined'&&S.busy){setState('thinking');return;}
|
|
startListening(token);
|
|
}
|
|
|
|
function onVoiceClick(event){
|
|
if(!ready) return;
|
|
event.preventDefault();
|
|
event.stopImmediatePropagation();
|
|
if(active) deactivate(true); else activate();
|
|
}
|
|
|
|
async function initialize(){
|
|
try{
|
|
const response=await fetch('/api/transcribe/capability',{cache:'no-store'});
|
|
const capability=await response.json().catch(function(){return {};});
|
|
if(!response.ok||!capability.available||capability.provider!=='local_command') return;
|
|
ready=true;
|
|
discoverStreamingCapability();
|
|
if(localStorage.getItem('hermes-atlas-voice-initialized')!=='1'){
|
|
localStorage.setItem('hermes-atlas-voice-initialized','1');
|
|
localStorage.setItem('hermes-voice-mode-button','true');
|
|
localStorage.setItem('hermes-tts-engine','atlas');
|
|
localStorage.setItem('hermes-tts-enabled','true');
|
|
}
|
|
// Move existing installations to the lower-latency silence window once;
|
|
// subsequent user changes remain untouched.
|
|
if(localStorage.getItem('hermes-atlas-voice-latency-v2')!=='1'){
|
|
localStorage.setItem('hermes-atlas-voice-latency-v2','1');
|
|
localStorage.setItem('hermes-voice-silence-ms','1100');
|
|
}
|
|
const selector=document.getElementById('settingsTtsEngine');
|
|
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
|
const option=document.createElement('option');
|
|
option.value='atlas';
|
|
option.textContent='Atlas Jetson (private)';
|
|
selector.insertBefore(option,selector.firstChild);
|
|
}
|
|
modeBtn.style.display=localStorage.getItem('hermes-voice-mode-button')==='false'?'none':'';
|
|
modeBtn.addEventListener('click',onVoiceClick,true);
|
|
window._applyVoiceModePref=function(){
|
|
if(typeof originalApplyPreference==='function') originalApplyPreference();
|
|
if(ready){
|
|
const enabled=localStorage.getItem('hermes-voice-mode-button')!=='false';
|
|
modeBtn.style.display=enabled?'':'none';
|
|
if(!enabled&&active) deactivate(false);
|
|
}
|
|
};
|
|
window.autoReadLastAssistant=function(){
|
|
if(active&&(state==='thinking'||state==='speaking')){speakResponse(generation);return;}
|
|
if(typeof originalAutoRead==='function') originalAutoRead.apply(this,arguments);
|
|
};
|
|
window._voiceModeActive=function(){return active;};
|
|
window._voiceModeDeactivate=function(){deactivate(false);};
|
|
window._voiceModeImmediateSend=function(){
|
|
if(active&&recorder&&recorder.state!=='inactive') recorder.stop();
|
|
};
|
|
}catch(_){
|
|
// The upstream browser voice implementation remains available as fallback.
|
|
}
|
|
}
|
|
|
|
initialize();
|
|
})();
|