323 lines
13 KiB
JavaScript
323 lines
13 KiB
JavaScript
// Natural turn-taking for chat.hermes.bstein.dev using the private Jetsons.
|
|
(function(){
|
|
'use strict';
|
|
|
|
const modeBtn=document.getElementById('btnVoiceMode');
|
|
const bar=document.getElementById('voiceModeBar');
|
|
const indicator=document.getElementById('voiceModeIndicator');
|
|
const label=document.getElementById('voiceModeLabel');
|
|
const composer=document.getElementById('msg');
|
|
if(!modeBtn||!bar||!indicator||!label||!composer||!navigator.mediaDevices||!window.MediaRecorder) return;
|
|
|
|
let ready=false;
|
|
let active=false;
|
|
let state='idle';
|
|
let generation=0;
|
|
let recorder=null;
|
|
let stream=null;
|
|
let audioContext=null;
|
|
let vadTimer=null;
|
|
let currentAudio=null;
|
|
let thinkingSession=null;
|
|
const originalAutoRead=window.autoReadLastAssistant;
|
|
const originalApplyPreference=window._applyVoiceModePref;
|
|
|
|
function toast(message){
|
|
if(typeof window.showToast==='function') window.showToast(message,3000);
|
|
}
|
|
|
|
function setState(next, customLabel){
|
|
state=next;
|
|
indicator.className='voice-mode-indicator '+next;
|
|
label.textContent=customLabel||(next==='listening'?'Listening…':next==='speaking'?'Speaking…':next==='thinking'?'Thinking…':'');
|
|
bar.style.display=active&&next!=='idle'?'':'none';
|
|
}
|
|
|
|
function stopCapture(){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
if(recorder&&recorder.state!=='inactive'){
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
recorder=null;
|
|
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
|
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
|
}
|
|
|
|
function stopPlayback(){
|
|
if(!currentAudio) return;
|
|
try{currentAudio.pause();currentAudio.currentTime=0;}catch(_){ }
|
|
currentAudio=null;
|
|
}
|
|
|
|
function deactivate(showMessage){
|
|
generation+=1;
|
|
active=false;
|
|
state='idle';
|
|
thinkingSession=null;
|
|
stopCapture();
|
|
stopPlayback();
|
|
modeBtn.classList.remove('active');
|
|
bar.style.display='none';
|
|
if(showMessage) toast('Hands-free voice mode off');
|
|
}
|
|
|
|
function restartSoon(token, delay){
|
|
window.setTimeout(function(){
|
|
if(active&&token===generation) startListening(token);
|
|
},delay||500);
|
|
}
|
|
|
|
function sendTranscript(transcript, token){
|
|
if(!active||token!==generation) return;
|
|
const text=String(transcript||'').trim();
|
|
if(!text){restartSoon(token,350);return;}
|
|
composer.value=text;
|
|
if(typeof window.autoResize==='function') window.autoResize();
|
|
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
setState('thinking');
|
|
if(typeof window.send==='function') window.send();
|
|
}
|
|
|
|
async function transcribe(blob, token){
|
|
if(!active||token!==generation) return;
|
|
setState('thinking','Transcribing…');
|
|
const ext=(blob.type||'').indexOf('ogg')>=0?'ogg':'webm';
|
|
const form=new FormData();
|
|
form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
|
|
try{
|
|
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
|
const payload=await response.json().catch(function(){return {};});
|
|
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
|
sendTranscript(payload.transcript,token);
|
|
}catch(error){
|
|
if(!active||token!==generation) return;
|
|
deactivate(false);
|
|
toast((error&&error.message)||'Private Whisper is unavailable');
|
|
// If the browser supplies its own recognizer, hand control back to the
|
|
// upstream voice implementation until the Jetson becomes healthy again.
|
|
if(window.SpeechRecognition||window.webkitSpeechRecognition){
|
|
modeBtn.removeEventListener('click',onVoiceClick,true);
|
|
window.setTimeout(function(){modeBtn.click();},50);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function startListening(token){
|
|
if(!active||token!==generation) return;
|
|
stopCapture();
|
|
setState('listening');
|
|
try{
|
|
const capture=await navigator.mediaDevices.getUserMedia({
|
|
audio:(function(){
|
|
const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
|
|
const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
|
|
if(supported.voiceIsolation) constraints.voiceIsolation=true;
|
|
return constraints;
|
|
})(),
|
|
});
|
|
if(!active||token!==generation){capture.getTracks().forEach(function(track){track.stop();});return;}
|
|
stream=capture;
|
|
const Context=window.AudioContext||window.webkitAudioContext;
|
|
audioContext=new Context();
|
|
const analyser=audioContext.createAnalyser();
|
|
analyser.fftSize=2048;
|
|
const highpass=audioContext.createBiquadFilter();
|
|
highpass.type='highpass';
|
|
highpass.frequency.value=140;
|
|
highpass.Q.value=0.7;
|
|
audioContext.createMediaStreamSource(stream).connect(highpass);
|
|
highpass.connect(analyser);
|
|
const samples=new Uint8Array(analyser.fftSize);
|
|
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm'];
|
|
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
|
const chunks=[];
|
|
const preRoll=[];
|
|
let heardSpeech=false;
|
|
let voiceFrames=0;
|
|
let noiseFloor=0.008;
|
|
let lastSpeech=Date.now();
|
|
const started=Date.now();
|
|
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
|
recorder.ondataavailable=function(event){
|
|
if(!event.data||!event.data.size) return;
|
|
if(heardSpeech){chunks.push(event.data);return;}
|
|
preRoll.push(event.data);
|
|
while(preRoll.length>3) preRoll.shift();
|
|
};
|
|
recorder.onstop=function(){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
const recordedStream=stream;
|
|
stream=null;
|
|
if(recordedStream) recordedStream.getTracks().forEach(function(track){track.stop();});
|
|
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
|
recorder=null;
|
|
if(!active||token!==generation) return;
|
|
if(!heardSpeech||!chunks.length){restartSoon(token,300);return;}
|
|
transcribe(new Blob(chunks,{type:mime||'audio/webm'}),token);
|
|
};
|
|
recorder.start(250);
|
|
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1600',10)||1600);
|
|
vadTimer=window.setInterval(function(){
|
|
if(!active||token!==generation||!recorder||recorder.state==='inactive') return;
|
|
analyser.getByteTimeDomainData(samples);
|
|
let energy=0;
|
|
for(let i=0;i<samples.length;i++){
|
|
const normalized=(samples[i]-128)/128;
|
|
energy+=normalized*normalized;
|
|
}
|
|
const rms=Math.sqrt(energy/samples.length);
|
|
const now=Date.now();
|
|
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
|
const voiceNow=rms>speechThreshold;
|
|
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
|
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
|
if(!heardSpeech&&voiceFrames>=3){
|
|
heardSpeech=true;
|
|
lastSpeech=now;
|
|
while(preRoll.length) chunks.push(preRoll.shift());
|
|
}else if(heardSpeech&&voiceNow){
|
|
lastSpeech=now;
|
|
}
|
|
const finished=heardSpeech&&(now-lastSpeech)>=silenceMs;
|
|
const timedOut=now-started>=90000;
|
|
const idle=(!heardSpeech)&&(now-started)>=20000;
|
|
if(finished||timedOut||idle){
|
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
|
try{recorder.stop();}catch(_){ }
|
|
}
|
|
},100);
|
|
}catch(error){
|
|
if(!active||token!==generation) return;
|
|
deactivate(false);
|
|
toast((error&&error.message)||'Microphone permission is required');
|
|
}
|
|
}
|
|
|
|
function cleanForSpeech(text){
|
|
if(typeof window._stripForTTS==='function') return window._stripForTTS(text);
|
|
return String(text||'').replace(/```[\s\S]*?```/g,' code block ').replace(/\s+/g,' ').trim();
|
|
}
|
|
|
|
function playBlob(blob, token){
|
|
return new Promise(function(resolve,reject){
|
|
if(!active||token!==generation){resolve();return;}
|
|
const url=URL.createObjectURL(blob);
|
|
const audio=new Audio(url);
|
|
currentAudio=audio;
|
|
function cleanup(){
|
|
if(currentAudio===audio) currentAudio=null;
|
|
URL.revokeObjectURL(url);
|
|
}
|
|
audio.onended=function(){cleanup();resolve();};
|
|
audio.onerror=function(){cleanup();reject(new Error('Local speech playback failed'));};
|
|
audio.play().catch(function(error){cleanup();reject(error);});
|
|
});
|
|
}
|
|
|
|
async function fetchSpeech(chunk){
|
|
const response=await fetch('/api/tts',{
|
|
method:'POST',
|
|
headers:{'Content-Type':'application/json'},
|
|
body:JSON.stringify({text:chunk,engine:'atlas'}),
|
|
});
|
|
if(!response.ok){
|
|
const payload=await response.json().catch(function(){return {};});
|
|
throw new Error(payload.error||('Local speech request failed: '+response.status));
|
|
}
|
|
return response.blob();
|
|
}
|
|
|
|
async function speakResponse(token){
|
|
if(!active||token!==generation) return;
|
|
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
|
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
|
thinkingSession=null;
|
|
restartSoon(token,250);
|
|
return;
|
|
}
|
|
thinkingSession=null;
|
|
const rows=document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
|
|
if(!rows.length){restartSoon(token,250);return;}
|
|
const text=cleanForSpeech(rows[rows.length-1].dataset.rawText||'');
|
|
if(!text){restartSoon(token,250);return;}
|
|
setState('speaking');
|
|
const chunks=typeof window._splitForTTS==='function'?window._splitForTTS(text,280):[text];
|
|
try{
|
|
let pending=fetchSpeech(chunks[0]);
|
|
for(let index=0;index<chunks.length;index+=1){
|
|
if(!active||token!==generation) return;
|
|
const blob=await pending;
|
|
if(index+1<chunks.length) pending=fetchSpeech(chunks[index+1]);
|
|
await playBlob(blob,token);
|
|
}
|
|
}catch(error){
|
|
if(active&&token===generation) toast((error&&error.message)||'Local speech is unavailable');
|
|
}
|
|
restartSoon(token,450);
|
|
}
|
|
|
|
function activate(){
|
|
generation+=1;
|
|
const token=generation;
|
|
active=true;
|
|
modeBtn.classList.add('active');
|
|
toast('Hands-free private voice mode on');
|
|
if(typeof window.stopTTS==='function') window.stopTTS();
|
|
if(typeof S!=='undefined'&&S.busy){setState('thinking');return;}
|
|
startListening(token);
|
|
}
|
|
|
|
function onVoiceClick(event){
|
|
if(!ready) return;
|
|
event.preventDefault();
|
|
event.stopImmediatePropagation();
|
|
if(active) deactivate(true); else activate();
|
|
}
|
|
|
|
async function initialize(){
|
|
try{
|
|
const response=await fetch('/api/transcribe/capability',{cache:'no-store'});
|
|
const capability=await response.json().catch(function(){return {};});
|
|
if(!response.ok||!capability.available||capability.provider!=='local_command') return;
|
|
ready=true;
|
|
if(localStorage.getItem('hermes-atlas-voice-initialized')!=='1'){
|
|
localStorage.setItem('hermes-atlas-voice-initialized','1');
|
|
localStorage.setItem('hermes-voice-mode-button','true');
|
|
localStorage.setItem('hermes-tts-engine','atlas');
|
|
localStorage.setItem('hermes-tts-enabled','true');
|
|
localStorage.setItem('hermes-voice-silence-ms','1600');
|
|
}
|
|
const selector=document.getElementById('settingsTtsEngine');
|
|
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
|
const option=document.createElement('option');
|
|
option.value='atlas';
|
|
option.textContent='Atlas Jetson (private)';
|
|
selector.insertBefore(option,selector.firstChild);
|
|
}
|
|
modeBtn.style.display=localStorage.getItem('hermes-voice-mode-button')==='false'?'none':'';
|
|
modeBtn.addEventListener('click',onVoiceClick,true);
|
|
window._applyVoiceModePref=function(){
|
|
if(typeof originalApplyPreference==='function') originalApplyPreference();
|
|
if(ready){
|
|
const enabled=localStorage.getItem('hermes-voice-mode-button')!=='false';
|
|
modeBtn.style.display=enabled?'':'none';
|
|
if(!enabled&&active) deactivate(false);
|
|
}
|
|
};
|
|
window.autoReadLastAssistant=function(){
|
|
if(active&&state==='thinking'){speakResponse(generation);return;}
|
|
if(typeof originalAutoRead==='function') originalAutoRead.apply(this,arguments);
|
|
};
|
|
window._voiceModeActive=function(){return active;};
|
|
window._voiceModeDeactivate=function(){deactivate(false);};
|
|
window._voiceModeImmediateSend=function(){
|
|
if(active&&recorder&&recorder.state!=='inactive') recorder.stop();
|
|
};
|
|
}catch(_){
|
|
// The upstream browser voice implementation remains available as fallback.
|
|
}
|
|
}
|
|
|
|
initialize();
|
|
})();
|