atlas-iac/dockerfiles/hermes-webui-atlas-voice.js

// Natural turn-taking for chat.hermes.bstein.dev using the private Jetsons.
(function(){
  'use strict';

  const modeBtn=document.getElementById('btnVoiceMode');
  const bar=document.getElementById('voiceModeBar');
  const indicator=document.getElementById('voiceModeIndicator');
  const label=document.getElementById('voiceModeLabel');
  const composer=document.getElementById('msg');
  if(!modeBtn||!bar||!indicator||!label||!composer||!navigator.mediaDevices||!window.MediaRecorder) return;

  let ready=false;
  let active=false;
  let state='idle';
  let generation=0;
  let recorder=null;
  let stream=null;
  let audioContext=null;
  let vadTimer=null;
  let currentAudio=null;
  let thinkingSession=null;
  // The only language signal this file trusts is the one the private Whisper
  // service returned for the audio of the turn currently being answered. It is
  // bound to that turn's generation token and consumed exactly once.
  let sttLanguage='';
  let sttLanguageToken=-1;
  const TTS_LANGUAGES=['en','ru','es'];
  const originalAutoRead=window.autoReadLastAssistant;
  const originalApplyPreference=window._applyVoiceModePref;

  function normalizeSttLanguage(value){
    if(typeof value!=='string') return '';
    const code=value.trim().toLowerCase();
    return TTS_LANGUAGES.indexOf(code)>=0?code:'';
  }

  function clearSttLanguage(){
    sttLanguage='';
    sttLanguageToken=-1;
  }

  function rememberSttLanguage(language, token){
    sttLanguage=language||'';
    sttLanguageToken=sttLanguage?token:-1;
  }

  function takeSttLanguage(token){
    const language=sttLanguageToken===token?sttLanguage:'';
    clearSttLanguage();
    return language;
  }

  function toast(message){
    if(typeof window.showToast==='function') window.showToast(message,3000);
  }

  function setState(next, customLabel){
    state=next;
    indicator.className='voice-mode-indicator '+next;
    label.textContent=customLabel||(next==='listening'?'Listening…':next==='speaking'?'Speaking…':next==='thinking'?'Thinking…':'');
    bar.style.display=active&&next!=='idle'?'':'none';
  }

  function stopCapture(){
    if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
    if(recorder&&recorder.state!=='inactive'){
      try{recorder.stop();}catch(_){ }
    }
    recorder=null;
    if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
    if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
  }

  function stopPlayback(){
    if(!currentAudio) return;
    try{currentAudio.pause();currentAudio.currentTime=0;}catch(_){ }
    currentAudio=null;
  }

  function deactivate(showMessage){
    generation+=1;
    active=false;
    state='idle';
    thinkingSession=null;
    clearSttLanguage();
    stopCapture();
    stopPlayback();
    modeBtn.classList.remove('active');
    bar.style.display='none';
    if(showMessage) toast('Hands-free voice mode off');
  }

  function restartSoon(token, delay){
    window.setTimeout(function(){
      if(active&&token===generation) startListening(token);
    },delay||500);
  }

  function sendTranscript(transcript, token, language){
    if(!active||token!==generation) return;
    const text=String(transcript||'').trim();
    if(!text){clearSttLanguage();restartSoon(token,350);return;}
    composer.value=text;
    if(typeof window.autoResize==='function') window.autoResize();
    thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
    rememberSttLanguage(language,token);
    setState('thinking');
    if(typeof window.send==='function') window.send();
  }

  async function transcribe(blob, token){
    if(!active||token!==generation) return;
    setState('thinking','Transcribing…');
    const ext=(blob.type||'').indexOf('ogg')>=0?'ogg':'webm';
    const form=new FormData();
    form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
    try{
      const response=await fetch('/api/transcribe',{method:'POST',body:form});
      const payload=await response.json().catch(function(){return {};});
      if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
      sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language));
    }catch(error){
      if(!active||token!==generation) return;
      deactivate(false);
      toast((error&&error.message)||'Private Whisper is unavailable');
      // If the browser supplies its own recognizer, hand control back to the
      // upstream voice implementation until the Jetson becomes healthy again.
      if(window.SpeechRecognition||window.webkitSpeechRecognition){
        modeBtn.removeEventListener('click',onVoiceClick,true);
        window.setTimeout(function(){modeBtn.click();},50);
      }
    }
  }

  async function startListening(token){
    if(!active||token!==generation) return;
    stopCapture();
    clearSttLanguage();
    setState('listening');
    try{
      const capture=await navigator.mediaDevices.getUserMedia({
        audio:(function(){
          const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
          const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
          if(supported.voiceIsolation) constraints.voiceIsolation=true;
          return constraints;
        })(),
      });
      if(!active||token!==generation){capture.getTracks().forEach(function(track){track.stop();});return;}
      stream=capture;
      const Context=window.AudioContext||window.webkitAudioContext;
      audioContext=new Context();
      const analyser=audioContext.createAnalyser();
      analyser.fftSize=2048;
      const highpass=audioContext.createBiquadFilter();
      highpass.type='highpass';
      highpass.frequency.value=140;
      highpass.Q.value=0.7;
      audioContext.createMediaStreamSource(stream).connect(highpass);
      highpass.connect(analyser);
      const samples=new Uint8Array(analyser.fftSize);
      const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm'];
      const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
      const chunks=[];
      const preRoll=[];
      let heardSpeech=false;
      let voiceFrames=0;
      let noiseFloor=0.008;
      let lastSpeech=Date.now();
      const started=Date.now();
      recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
      recorder.ondataavailable=function(event){
        if(!event.data||!event.data.size) return;
        if(heardSpeech){chunks.push(event.data);return;}
        preRoll.push(event.data);
        while(preRoll.length>3) preRoll.shift();
      };
      recorder.onstop=function(){
        if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
        const recordedStream=stream;
        stream=null;
        if(recordedStream) recordedStream.getTracks().forEach(function(track){track.stop();});
        if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
        recorder=null;
        if(!active||token!==generation) return;
        if(!heardSpeech||!chunks.length){restartSoon(token,300);return;}
        transcribe(new Blob(chunks,{type:mime||'audio/webm'}),token);
      };
      recorder.start(250);
      const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1600',10)||1600);
      vadTimer=window.setInterval(function(){
        if(!active||token!==generation||!recorder||recorder.state==='inactive') return;
        analyser.getByteTimeDomainData(samples);
        let energy=0;
        for(let i=0;i<samples.length;i++){
          const normalized=(samples[i]-128)/128;
          energy+=normalized*normalized;
        }
        const rms=Math.sqrt(energy/samples.length);
        const now=Date.now();
        const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
        const voiceNow=rms>speechThreshold;
        if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
        voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
        if(!heardSpeech&&voiceFrames>=3){
          heardSpeech=true;
          lastSpeech=now;
          while(preRoll.length) chunks.push(preRoll.shift());
        }else if(heardSpeech&&voiceNow){
          lastSpeech=now;
        }
        const finished=heardSpeech&&(now-lastSpeech)>=silenceMs;
        const timedOut=now-started>=90000;
        const idle=(!heardSpeech)&&(now-started)>=20000;
        if(finished||timedOut||idle){
          if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
          try{recorder.stop();}catch(_){ }
        }
      },100);
    }catch(error){
      if(!active||token!==generation) return;
      deactivate(false);
      toast((error&&error.message)||'Microphone permission is required');
    }
  }

  function cleanForSpeech(text){
    if(typeof window._stripForTTS==='function') return window._stripForTTS(text);
    return String(text||'').replace(/```[\s\S]*?```/g,' code block ').replace(/\s+/g,' ').trim();
  }

  function playBlob(blob, token){
    return new Promise(function(resolve,reject){
      if(!active||token!==generation){resolve();return;}
      const url=URL.createObjectURL(blob);
      const audio=new Audio(url);
      currentAudio=audio;
      function cleanup(){
        if(currentAudio===audio) currentAudio=null;
        URL.revokeObjectURL(url);
      }
      audio.onended=function(){cleanup();resolve();};
      audio.onerror=function(){cleanup();reject(new Error('Local speech playback failed'));};
      audio.play().catch(function(error){cleanup();reject(error);});
    });
  }

  async function fetchSpeech(chunk, language){
    // `language` is only ever the private STT result for this turn. When it is
    // absent the field is omitted entirely and the server picks English.
    const request={text:chunk,engine:'atlas'};
    if(language) request.language=language;
    const response=await fetch('/api/tts',{
      method:'POST',
      headers:{'Content-Type':'application/json'},
      body:JSON.stringify(request),
    });
    if(!response.ok){
      const payload=await response.json().catch(function(){return {};});
      throw new Error(payload.error||('Local speech request failed: '+response.status));
    }
    return response.blob();
  }

  async function speakResponse(token){
    if(!active||token!==generation) return;
    const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
    if(thinkingSession&&currentSession&&thinkingSession!==currentSession){
      thinkingSession=null;
      clearSttLanguage();
      restartSoon(token,250);
      return;
    }
    thinkingSession=null;
    const language=takeSttLanguage(token);
    const rows=document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
    if(!rows.length){restartSoon(token,250);return;}
    const text=cleanForSpeech(rows[rows.length-1].dataset.rawText||'');
    if(!text){restartSoon(token,250);return;}
    setState('speaking');
    const chunks=typeof window._splitForTTS==='function'?window._splitForTTS(text,280):[text];
    try{
      let pending=fetchSpeech(chunks[0],language);
      for(let index=0;index<chunks.length;index+=1){
        if(!active||token!==generation) return;
        const blob=await pending;
        if(index+1<chunks.length) pending=fetchSpeech(chunks[index+1],language);
        await playBlob(blob,token);
      }
    }catch(error){
      if(active&&token===generation) toast((error&&error.message)||'Local speech is unavailable');
    }
    restartSoon(token,450);
  }

  function activate(){
    generation+=1;
    const token=generation;
    active=true;
    clearSttLanguage();
    modeBtn.classList.add('active');
    toast('Hands-free private voice mode on');
    if(typeof window.stopTTS==='function') window.stopTTS();
    if(typeof S!=='undefined'&&S.busy){setState('thinking');return;}
    startListening(token);
  }

  function onVoiceClick(event){
    if(!ready) return;
    event.preventDefault();
    event.stopImmediatePropagation();
    if(active) deactivate(true); else activate();
  }

  async function initialize(){
    try{
      const response=await fetch('/api/transcribe/capability',{cache:'no-store'});
      const capability=await response.json().catch(function(){return {};});
      if(!response.ok||!capability.available||capability.provider!=='local_command') return;
      ready=true;
      if(localStorage.getItem('hermes-atlas-voice-initialized')!=='1'){
        localStorage.setItem('hermes-atlas-voice-initialized','1');
        localStorage.setItem('hermes-voice-mode-button','true');
        localStorage.setItem('hermes-tts-engine','atlas');
        localStorage.setItem('hermes-tts-enabled','true');
        localStorage.setItem('hermes-voice-silence-ms','1600');
      }
      const selector=document.getElementById('settingsTtsEngine');
      if(selector&&!selector.querySelector('option[value="atlas"]')){
        const option=document.createElement('option');
        option.value='atlas';
        option.textContent='Atlas Jetson (private)';
        selector.insertBefore(option,selector.firstChild);
      }
      modeBtn.style.display=localStorage.getItem('hermes-voice-mode-button')==='false'?'none':'';
      modeBtn.addEventListener('click',onVoiceClick,true);
      window._applyVoiceModePref=function(){
        if(typeof originalApplyPreference==='function') originalApplyPreference();
        if(ready){
          const enabled=localStorage.getItem('hermes-voice-mode-button')!=='false';
          modeBtn.style.display=enabled?'':'none';
          if(!enabled&&active) deactivate(false);
        }
      };
      window.autoReadLastAssistant=function(){
        if(active&&state==='thinking'){speakResponse(generation);return;}
        if(typeof originalAutoRead==='function') originalAutoRead.apply(this,arguments);
      };
      window._voiceModeActive=function(){return active;};
      window._voiceModeDeactivate=function(){deactivate(false);};
      window._voiceModeImmediateSend=function(){
        if(active&&recorder&&recorder.state!=='inactive') recorder.stop();
      };
    }catch(_){
      // The upstream browser voice implementation remains available as fallback.
    }
  }

  initialize();
})();