hermes(voice): continuous mic, barge stitching, 1.15x speech
Three conversational fixes for hands-free chat: - The microphone now stays hot for the whole session: capture runs on its own epoch, re-arms immediately after each utterance endpoints, and keeps recording through transcribing/thinking/speaking - speech is never lost to Hermes being busy. Speech onset during a response cancels it through the live capture path (echo-guarded exactly like the old monitor) without touching the running recorder. - When the user talks over Hermes before any visible reply appeared, the interrupted utterance and the follow-up are stitched into one message (20s window), so the response addresses the whole thought. - TTS speaks 15% faster by default (server-side length_scale, no pitch shift), user-tunable via hermes-voice-tts-speed (0.5-2.0), honored on streaming, WAV fallback and thinking-cue paths. 245 voice-lane tests pass; single getUserMedia site preserved; Dockerfile grep guards verified. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
This commit is contained in:
parent
17037773bf
commit
91cfb901a2
@ -456,6 +456,25 @@ def _handle_atlas_voice_preflight(handler):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _atlas_tts_speed(body):
|
||||||
|
"""Clamp the optional client speech-rate to Piper's supported 0.5-2.0.
|
||||||
|
|
||||||
|
The value is a UX preference, not a trust decision: a missing, boolean,
|
||||||
|
non-numeric or NaN value falls back to the neutral 1.0 this proxy always
|
||||||
|
sent before the hands-free speed became client-tunable. Infinities clamp
|
||||||
|
to the range bounds like any other out-of-range number.
|
||||||
|
"""
|
||||||
|
if not isinstance(body, dict):
|
||||||
|
return 1.0
|
||||||
|
value = body.get("speed")
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||||
|
return 1.0
|
||||||
|
speed = float(value)
|
||||||
|
if speed != speed:
|
||||||
|
return 1.0
|
||||||
|
return max(0.5, min(2.0, speed))
|
||||||
|
|
||||||
|
|
||||||
def _atlas_tts_stream_payload(data):
|
def _atlas_tts_stream_payload(data):
|
||||||
"""Build the narrow Piper payload used by the raw-PCM stream endpoint."""
|
"""Build the narrow Piper payload used by the raw-PCM stream endpoint."""
|
||||||
if not isinstance(data, dict):
|
if not isinstance(data, dict):
|
||||||
@ -466,7 +485,7 @@ def _atlas_tts_stream_payload(data):
|
|||||||
text = text.strip()
|
text = text.strip()
|
||||||
if len(text) > 500:
|
if len(text) > 500:
|
||||||
raise ValueError("text too long (max 500 characters)")
|
raise ValueError("text too long (max 500 characters)")
|
||||||
payload = {"model": "piper", "input": text, "speed": 1.0}
|
payload = {"model": "piper", "input": text, "speed": _atlas_tts_speed(data)}
|
||||||
language = _atlas_tts_language(data)
|
language = _atlas_tts_language(data)
|
||||||
if language:
|
if language:
|
||||||
payload["language"] = language
|
payload["language"] = language
|
||||||
@ -745,6 +764,10 @@ atlas = ''' # ── Atlas private Jetson TTS ──────────
|
|||||||
speed = max(0.5, min(2.0, 1.0 + (float(rate_str.rstrip("%")) / 100.0)))
|
speed = max(0.5, min(2.0, 1.0 + (float(rate_str.rstrip("%")) / 100.0)))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
speed = 1.0
|
speed = 1.0
|
||||||
|
if isinstance(data, dict) and "speed" in data:
|
||||||
|
# Hands-free clients send an explicit validated speed; it wins
|
||||||
|
# over the legacy percentage rate string.
|
||||||
|
speed = _atlas_tts_speed(data)
|
||||||
request_payload = {
|
request_payload = {
|
||||||
"model": "piper",
|
"model": "piper",
|
||||||
"input": text,
|
"input": text,
|
||||||
|
|||||||
@ -36,6 +36,17 @@
|
|||||||
let visualInputLevel=0;
|
let visualInputLevel=0;
|
||||||
let streamingCapability={tts:null,stt:null,preflight:null};
|
let streamingCapability={tts:null,stt:null,preflight:null};
|
||||||
let voicePreflight=null;
|
let voicePreflight=null;
|
||||||
|
// Continuous-capture state. The microphone stream and its AudioContext stay
|
||||||
|
// hot for the whole hands-free session; each utterance is one "capture turn"
|
||||||
|
// guarded by captureGeneration so a response-side barge (which bumps
|
||||||
|
// `generation`) never interrupts the running microphone pipeline.
|
||||||
|
let captureGeneration=0;
|
||||||
|
let captureActive=false;
|
||||||
|
let captureGraph=null;
|
||||||
|
// Barge-in stitching: transcript whose model response was cancelled before
|
||||||
|
// any visible assistant output, plus the transcript most recently sent.
|
||||||
|
let pendingStitch=null;
|
||||||
|
let lastSentTranscript=null;
|
||||||
const voiceTabNonce=(function(){
|
const voiceTabNonce=(function(){
|
||||||
try{
|
try{
|
||||||
const bytes=new Uint8Array(16);
|
const bytes=new Uint8Array(16);
|
||||||
@ -58,6 +69,12 @@
|
|||||||
const BARGE_LOOKBACK_MS=900;
|
const BARGE_LOOKBACK_MS=900;
|
||||||
const BARGE_DUCK_FRAMES=2;
|
const BARGE_DUCK_FRAMES=2;
|
||||||
const BARGE_TRIGGER_FRAMES=4;
|
const BARGE_TRIGGER_FRAMES=4;
|
||||||
|
// A response cancelled by barge-in leaves the user mid-thought: the next
|
||||||
|
// finalized utterance inside this window is sent as one stitched message.
|
||||||
|
const STITCH_WINDOW_MS=20000;
|
||||||
|
const TTS_SPEED_DEFAULT=1.15;
|
||||||
|
const TTS_SPEED_MIN=0.5;
|
||||||
|
const TTS_SPEED_MAX=2;
|
||||||
const THINKING_CUE_FIRST_MS=1900;
|
const THINKING_CUE_FIRST_MS=1900;
|
||||||
const THINKING_CUE_INTERVAL_MS=6500;
|
const THINKING_CUE_INTERVAL_MS=6500;
|
||||||
const THINKING_CUE_POOLS={
|
const THINKING_CUE_POOLS={
|
||||||
@ -304,11 +321,13 @@
|
|||||||
thinkingSession=null;
|
thinkingSession=null;
|
||||||
thinkingTurnId='';
|
thinkingTurnId='';
|
||||||
suppressAutoRead=false;
|
suppressAutoRead=false;
|
||||||
|
pendingStitch=null;
|
||||||
|
lastSentTranscript=null;
|
||||||
clearBargeCancellation();
|
clearBargeCancellation();
|
||||||
stopResponseObserver();
|
stopResponseObserver();
|
||||||
cancelThinkingCues();
|
cancelThinkingCues();
|
||||||
cancelSpeechTurn();
|
cancelSpeechTurn();
|
||||||
stopCapture();
|
releaseMicrophone();
|
||||||
stopPlayback();
|
stopPlayback();
|
||||||
modeBtn.classList.remove('active');
|
modeBtn.classList.remove('active');
|
||||||
setState('error',message);
|
setState('error',message);
|
||||||
@ -320,6 +339,13 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function stopCapture(){
|
function stopCapture(){
|
||||||
|
// Tear down the per-utterance capture pipeline (recorder, VAD, streaming
|
||||||
|
// STT, WebAudio graph) but deliberately KEEP the microphone stream and its
|
||||||
|
// AudioContext, so the next capture turn starts instantly and speech
|
||||||
|
// during transcribing/thinking is never lost. Only releaseMicrophone()
|
||||||
|
// actually ends the session's single getUserMedia lease.
|
||||||
|
captureGeneration+=1;
|
||||||
|
captureActive=false;
|
||||||
cancelVoicePreflight(captureTurnId);
|
cancelVoicePreflight(captureTurnId);
|
||||||
stopBargeMonitor();
|
stopBargeMonitor();
|
||||||
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
||||||
@ -327,8 +353,24 @@
|
|||||||
try{recorder.stop();}catch(_){ }
|
try{recorder.stop();}catch(_){ }
|
||||||
}
|
}
|
||||||
recorder=null;
|
recorder=null;
|
||||||
|
if(captureNode){try{captureNode.disconnect();}catch(_){ }}
|
||||||
captureNode=null;
|
captureNode=null;
|
||||||
|
if(captureGraph){
|
||||||
|
// The AudioContext persists across turns, so every per-turn node must be
|
||||||
|
// detached here or the shared graph would grow with each utterance.
|
||||||
|
try{if(captureGraph.mediaSource) captureGraph.mediaSource.disconnect();}catch(_){ }
|
||||||
|
try{if(captureGraph.highpass) captureGraph.highpass.disconnect();}catch(_){ }
|
||||||
|
try{if(captureGraph.analyser) captureGraph.analyser.disconnect();}catch(_){ }
|
||||||
|
try{if(captureGraph.silentGain) captureGraph.silentGain.disconnect();}catch(_){ }
|
||||||
|
captureGraph=null;
|
||||||
|
}
|
||||||
cancelStreamingStt();
|
cancelStreamingStt();
|
||||||
|
}
|
||||||
|
|
||||||
|
function releaseMicrophone(){
|
||||||
|
// Full microphone teardown. Used only by deactivate(), showUnavailable()
|
||||||
|
// and fatal capture errors; every other path retains the hot microphone.
|
||||||
|
stopCapture();
|
||||||
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
||||||
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
||||||
}
|
}
|
||||||
@ -369,11 +411,13 @@
|
|||||||
clearErrorTimer();
|
clearErrorTimer();
|
||||||
clearSttLanguage();
|
clearSttLanguage();
|
||||||
suppressAutoRead=false;
|
suppressAutoRead=false;
|
||||||
|
pendingStitch=null;
|
||||||
|
lastSentTranscript=null;
|
||||||
clearBargeCancellation();
|
clearBargeCancellation();
|
||||||
stopResponseObserver();
|
stopResponseObserver();
|
||||||
cancelThinkingCues();
|
cancelThinkingCues();
|
||||||
cancelSpeechTurn();
|
cancelSpeechTurn();
|
||||||
stopCapture();
|
releaseMicrophone();
|
||||||
stopPlayback();
|
stopPlayback();
|
||||||
modeBtn.classList.remove('active');
|
modeBtn.classList.remove('active');
|
||||||
setState('idle');
|
setState('idle');
|
||||||
@ -382,7 +426,12 @@
|
|||||||
|
|
||||||
function restartSoon(token,delay){
|
function restartSoon(token,delay){
|
||||||
window.setTimeout(function(){
|
window.setTimeout(function(){
|
||||||
if(active&&token===generation) startListening(token);
|
if(!active||token!==generation) return;
|
||||||
|
// Continuous capture: while the microphone pipeline is already running,
|
||||||
|
// finishing a response only needs the display returned to Listening.
|
||||||
|
// Rebuilding capture here would drop speech already being collected.
|
||||||
|
if(captureActive){setState('listening');return;}
|
||||||
|
startListening(token);
|
||||||
},delay||500);
|
},delay||500);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -422,31 +471,56 @@
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function sendTranscript(transcript,token,language){
|
function recordPendingStitch(){
|
||||||
|
// Called at the moment a model turn is cancelled by user speech. Stitching
|
||||||
|
// only applies when the cancelled turn produced NO visible assistant text:
|
||||||
|
// the model never answered, so the next utterance restates the complete
|
||||||
|
// thought as one message. Any partial answer means the interruption stands
|
||||||
|
// on its own and the next utterance is sent alone.
|
||||||
|
const sent=lastSentTranscript;
|
||||||
|
lastSentTranscript=null;
|
||||||
|
if(!sent||!sent.text) return;
|
||||||
|
if(currentAssistantText()){pendingStitch=null;return;}
|
||||||
|
pendingStitch={text:sent.text,at:Date.now()};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendTranscript(transcript,token,language,turnId){
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
cancelVoicePreflight(captureTurnId);
|
cancelVoicePreflight(turnId||captureTurnId);
|
||||||
const text=String(transcript||'').trim();
|
const text=String(transcript||'').trim();
|
||||||
if(!text){clearSttLanguage();restartSoon(token,350);return;}
|
if(!text){clearSttLanguage();restartSoon(token,350);return;}
|
||||||
composer.value=text;
|
|
||||||
if(typeof window.autoResize==='function') window.autoResize();
|
|
||||||
setState('thinking');
|
|
||||||
startBargeMonitor(token);
|
|
||||||
if(!bargeCancelPromise&&typeof S!=='undefined'&&(S.busy||S.activeStreamId)){
|
if(!bargeCancelPromise&&typeof S!=='undefined'&&(S.busy||S.activeStreamId)){
|
||||||
|
// A new utterance finished while the previous model turn was still in
|
||||||
|
// flight (continuous capture makes this a normal interruption): remember
|
||||||
|
// the interrupted transcript for stitching, then cancel the stale turn.
|
||||||
|
recordPendingStitch();
|
||||||
suppressAutoRead=true;
|
suppressAutoRead=true;
|
||||||
bargeCancelPromise=cancelActiveModelTurn();
|
bargeCancelPromise=cancelActiveModelTurn();
|
||||||
}
|
}
|
||||||
|
// A turn whose response was barge-cancelled before any assistant output is
|
||||||
|
// restated as one stitched message, so the model answers the full thought.
|
||||||
|
const stitch=(pendingStitch&&pendingStitch.text&&(Date.now()-pendingStitch.at)<=STITCH_WINDOW_MS)?pendingStitch:null;
|
||||||
|
pendingStitch=null;
|
||||||
|
composer.value=stitch?stitch.text+' '+text:text;
|
||||||
|
if(typeof window.autoResize==='function') window.autoResize();
|
||||||
|
setState('thinking');
|
||||||
|
// The live VAD/STT capture is the barge-in detector while it runs; the
|
||||||
|
// energy-only monitor remains only as a fallback when capture is down.
|
||||||
|
if(!captureActive) startBargeMonitor(token);
|
||||||
const cancellationSettled=await settleBargeCancellation(token);
|
const cancellationSettled=await settleBargeCancellation(token);
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
if(!cancellationSettled){
|
if(!cancellationSettled){
|
||||||
|
pendingStitch=stitch;
|
||||||
toast('The previous response did not stop. Please repeat your interruption.');
|
toast('The previous response did not stop. Please repeat your interruption.');
|
||||||
restartSoon(token,250);
|
restartSoon(token,250);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
||||||
thinkingTurnId=captureTurnId||String(token)+'-'+String(++turnSequence);
|
thinkingTurnId=turnId||captureTurnId||String(token)+'-'+String(++turnSequence);
|
||||||
rememberAssistantBaseline();
|
rememberAssistantBaseline();
|
||||||
rememberSttLanguage(language,token);
|
rememberSttLanguage(language,token);
|
||||||
if(typeof window.send==='function'){
|
if(typeof window.send==='function'){
|
||||||
|
lastSentTranscript={text:composer.value,token:token};
|
||||||
window.send();
|
window.send();
|
||||||
suppressAutoRead=false;
|
suppressAutoRead=false;
|
||||||
startResponseObserver(token);
|
startResponseObserver(token);
|
||||||
@ -462,7 +536,7 @@
|
|||||||
return 'webm';
|
return 'webm';
|
||||||
}
|
}
|
||||||
|
|
||||||
async function transcribe(blob,token){
|
async function transcribe(blob,token,turnId){
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
setState('transcribing');
|
setState('transcribing');
|
||||||
const ext=audioExtension(blob.type);
|
const ext=audioExtension(blob.type);
|
||||||
@ -472,7 +546,7 @@
|
|||||||
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
||||||
const payload=await response.json().catch(function(){return {};});
|
const payload=await response.json().catch(function(){return {};});
|
||||||
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
||||||
sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language));
|
sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language),turnId);
|
||||||
}catch(error){
|
}catch(error){
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
const message=errorMessage(error,'Private Whisper is unavailable');
|
const message=errorMessage(error,'Private Whisper is unavailable');
|
||||||
@ -754,6 +828,7 @@
|
|||||||
node.connect(silentGain);
|
node.connect(silentGain);
|
||||||
silentGain.connect(context.destination);
|
silentGain.connect(context.destination);
|
||||||
captureNode=node;
|
captureNode=node;
|
||||||
|
if(captureGraph) captureGraph.silentGain=silentGain;
|
||||||
session.setWorkletReady(true);
|
session.setWorkletReady(true);
|
||||||
return true;
|
return true;
|
||||||
}catch(_){
|
}catch(_){
|
||||||
@ -811,6 +886,7 @@
|
|||||||
// during AudioWorklet/session setup.
|
// during AudioWorklet/session setup.
|
||||||
monitor.handoff=true;
|
monitor.handoff=true;
|
||||||
const oldToken=monitor.token;
|
const oldToken=monitor.token;
|
||||||
|
recordPendingStitch();
|
||||||
generation+=1;
|
generation+=1;
|
||||||
const token=generation;
|
const token=generation;
|
||||||
thinkingSession=null;
|
thinkingSession=null;
|
||||||
@ -836,6 +912,30 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function bargeFromLiveCapture(){
|
||||||
|
// Live-capture barge-in: the always-on VAD/STT listener detected a real
|
||||||
|
// speech onset while Hermes was mid-response. Cancel the RESPONSE side
|
||||||
|
// only — cues, playback, the model stream — and bump `generation` so every
|
||||||
|
// stale response-side continuation dies. The already-running capture is
|
||||||
|
// guarded by captureGeneration and keeps collecting the interrupting
|
||||||
|
// utterance without losing a syllable.
|
||||||
|
if(!active||(state!=='thinking'&&state!=='speaking')) return;
|
||||||
|
recordPendingStitch();
|
||||||
|
generation+=1;
|
||||||
|
thinkingSession=null;
|
||||||
|
thinkingTurnId='';
|
||||||
|
suppressAutoRead=true;
|
||||||
|
clearSttLanguage();
|
||||||
|
stopResponseObserver();
|
||||||
|
cancelThinkingCues();
|
||||||
|
cancelSpeechTurn();
|
||||||
|
stopPlayback();
|
||||||
|
if(typeof window.stopTTS==='function') window.stopTTS();
|
||||||
|
clearBargeCancellation();
|
||||||
|
bargeCancelPromise=cancelActiveModelTurn();
|
||||||
|
setState('listening','Listening — interrupted');
|
||||||
|
}
|
||||||
|
|
||||||
async function startBargeMonitor(token){
|
async function startBargeMonitor(token){
|
||||||
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')||bargeMonitor) return;
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')||bargeMonitor) return;
|
||||||
const monitor={token:token,cancelled:false,handoff:false,aecUsable:true,timer:null,stream:null,context:null,source:null,node:null,silentGain:null,lookback:[],lookbackFrames:0};
|
const monitor={token:token,cancelled:false,handoff:false,aecUsable:true,timer:null,stream:null,context:null,source:null,node:null,silentGain:null,lookback:[],lookbackFrames:0};
|
||||||
@ -915,7 +1015,7 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function transcribeStreamingOrFallback(blob,token,session,allowContainerFallback){
|
async function transcribeStreamingOrFallback(blob,token,session,allowContainerFallback,turnId){
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
setState('transcribing');
|
setState('transcribing');
|
||||||
let pcmFallback=null;
|
let pcmFallback=null;
|
||||||
@ -924,8 +1024,8 @@
|
|||||||
const result=await session.commit();
|
const result=await session.commit();
|
||||||
if(!String(result.transcript||'').trim()) throw new Error('Streaming transcription returned no final text');
|
if(!String(result.transcript||'').trim()) throw new Error('Streaming transcription returned no final text');
|
||||||
if(active&&token===generation){
|
if(active&&token===generation){
|
||||||
streamingStt=null;
|
if(streamingStt===session) streamingStt=null;
|
||||||
sendTranscript(result.transcript,token,result.language);
|
sendTranscript(result.transcript,token,result.language,turnId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}catch(_){
|
}catch(_){
|
||||||
@ -936,7 +1036,7 @@
|
|||||||
if(streamingStt===session) streamingStt=null;
|
if(streamingStt===session) streamingStt=null;
|
||||||
}
|
}
|
||||||
if(pcmFallback){
|
if(pcmFallback){
|
||||||
transcribe(pcmFallback,token);
|
transcribe(pcmFallback,token,turnId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(allowContainerFallback===false){
|
if(allowContainerFallback===false){
|
||||||
@ -944,29 +1044,54 @@
|
|||||||
restartSoon(token,250);
|
restartSoon(token,250);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transcribe(blob,token);
|
transcribe(blob,token,turnId);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function startListening(token,reusedCapture){
|
async function startListening(token,reusedCapture){
|
||||||
if(!active||token!==generation) return;
|
if(!active||token!==generation) return;
|
||||||
stopCapture();
|
stopCapture();
|
||||||
stopPlayback();
|
// A continuous-capture restart (reusedCapture.preserveDisplay) happens the
|
||||||
cancelSpeechTurn();
|
// instant an utterance finalizes, while the display legitimately shows
|
||||||
clearSttLanguage();
|
// Hermes's own activity (transcribing/thinking/speaking). Capture is
|
||||||
|
// therefore tracked by captureActive, never by the visible state, and the
|
||||||
|
// response-side teardown below is skipped so that turn is not disturbed.
|
||||||
|
const preserveDisplay=!!(reusedCapture&&reusedCapture.preserveDisplay);
|
||||||
|
if(!preserveDisplay){
|
||||||
|
stopPlayback();
|
||||||
|
cancelSpeechTurn();
|
||||||
|
clearSttLanguage();
|
||||||
|
}
|
||||||
captureTurnId=(voiceTabNonce?voiceTabNonce+'-':'')+String(token)+'-'+String(++turnSequence);
|
captureTurnId=(voiceTabNonce?voiceTabNonce+'-':'')+String(token)+'-'+String(++turnSequence);
|
||||||
setState('listening');
|
// Capture turns carry their own epoch: a response-side barge bumps
|
||||||
|
// `generation` but never this counter, so capture survives it seamlessly.
|
||||||
|
const captureToken=++captureGeneration;
|
||||||
|
captureActive=true;
|
||||||
|
if(!preserveDisplay) setState('listening');
|
||||||
try{
|
try{
|
||||||
const capture=reusedCapture&&reusedCapture.stream?reusedCapture.stream:await acquireMicrophone();
|
// A barge-monitor handoff supplies its own stream/context; otherwise the
|
||||||
if(!active||token!==generation){
|
// retained session microphone is reused. Only a fresh hands-free session
|
||||||
|
// actually asks for the microphone again, keeping acquireMicrophone()
|
||||||
|
// the single getUserMedia call site.
|
||||||
|
if(reusedCapture&&reusedCapture.stream&&stream&&reusedCapture.stream!==stream){
|
||||||
|
stream.getTracks().forEach(function(track){track.stop();});
|
||||||
|
stream=null;
|
||||||
|
}
|
||||||
|
if(reusedCapture&&reusedCapture.context&&audioContext&&reusedCapture.context!==audioContext){
|
||||||
|
try{audioContext.close();}catch(_){ }
|
||||||
|
audioContext=null;
|
||||||
|
}
|
||||||
|
const capture=(reusedCapture&&reusedCapture.stream)||stream||await acquireMicrophone();
|
||||||
|
if(!active||captureToken!==captureGeneration){
|
||||||
if(reusedCapture&&reusedCapture.handoffMonitor) disposeBargeResources(reusedCapture.handoffMonitor,false);
|
if(reusedCapture&&reusedCapture.handoffMonitor) disposeBargeResources(reusedCapture.handoffMonitor,false);
|
||||||
else capture.getTracks().forEach(function(track){track.stop();});
|
else if(capture!==stream) capture.getTracks().forEach(function(track){track.stop();});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
stream=capture;
|
stream=capture;
|
||||||
|
const captureAec=captureAecIsUsable(stream);
|
||||||
const Context=window.AudioContext||window.webkitAudioContext;
|
const Context=window.AudioContext||window.webkitAudioContext;
|
||||||
if(reusedCapture&&reusedCapture.context){
|
if(reusedCapture&&reusedCapture.context){
|
||||||
audioContext=reusedCapture.context;
|
audioContext=reusedCapture.context;
|
||||||
}else{
|
}else if(!audioContext){
|
||||||
try{
|
try{
|
||||||
// Let the browser's native resampler produce Whisper's 16 kHz input.
|
// Let the browser's native resampler produce Whisper's 16 kHz input.
|
||||||
audioContext=new Context({sampleRate:16000,latencyHint:'interactive'});
|
audioContext=new Context({sampleRate:16000,latencyHint:'interactive'});
|
||||||
@ -983,6 +1108,7 @@
|
|||||||
const mediaSource=audioContext.createMediaStreamSource(stream);
|
const mediaSource=audioContext.createMediaStreamSource(stream);
|
||||||
mediaSource.connect(highpass);
|
mediaSource.connect(highpass);
|
||||||
highpass.connect(analyser);
|
highpass.connect(analyser);
|
||||||
|
captureGraph={mediaSource:mediaSource,highpass:highpass,analyser:analyser,silentGain:null};
|
||||||
streamingStt=createStreamingSttSession(captureTurnId,audioContext);
|
streamingStt=createStreamingSttSession(captureTurnId,audioContext);
|
||||||
// Whisper receives the browser's full-band processed microphone signal;
|
// Whisper receives the browser's full-band processed microphone signal;
|
||||||
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
|
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
|
||||||
@ -1010,6 +1136,9 @@
|
|||||||
let noiseFloor=0.008;
|
let noiseFloor=0.008;
|
||||||
let lastSpeech=Date.now();
|
let lastSpeech=Date.now();
|
||||||
let speculative=false;
|
let speculative=false;
|
||||||
|
let ducked=false;
|
||||||
|
let bargeArmAt=0;
|
||||||
|
let playbackWasLive=false;
|
||||||
const started=Date.now();
|
const started=Date.now();
|
||||||
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
||||||
let recordedMime=recorder.mimeType||mime||'';
|
let recordedMime=recorder.mimeType||mime||'';
|
||||||
@ -1027,23 +1156,39 @@
|
|||||||
new Promise(function(resolve){window.setTimeout(resolve,100);}),
|
new Promise(function(resolve){window.setTimeout(resolve,100);}),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
const recordedStream=stream;
|
// A stale onstop (teardown by stopCapture/releaseMicrophone, or a
|
||||||
stream=null;
|
// newer capture turn already running) must not touch the globals: by
|
||||||
if(recordedStream) recordedStream.getTracks().forEach(function(track){track.stop();});
|
// now they can belong to the NEXT turn, and its session was already
|
||||||
const context=audioContext;
|
// cancelled by stopCapture. Real recorders fire onstop asynchronously,
|
||||||
audioContext=null;
|
// so this guard has to come before anything else is read.
|
||||||
if(context){try{context.close();}catch(_){ }}
|
if(!active||captureToken!==captureGeneration) return;
|
||||||
captureNode=null;
|
// Continuous capture: the microphone stream and AudioContext stay hot.
|
||||||
recorder=null;
|
// Detach only this utterance's session, then re-enter capture below so
|
||||||
if(!active||token!==generation){cancelStreamingStt();return;}
|
// speech during transcribing/thinking becomes the next utterance.
|
||||||
if(!heardSpeech||!chunks.length){cancelStreamingStt();restartSoon(token,300);return;}
|
const utteranceTurnId=captureTurnId;
|
||||||
const session=streamingStt;
|
const session=streamingStt;
|
||||||
|
streamingStt=null;
|
||||||
|
recorder=null;
|
||||||
|
if(!heardSpeech||!chunks.length){
|
||||||
|
if(session) session.cancel();
|
||||||
|
// Nothing worth transcribing: recycle the recorder on the hot mic.
|
||||||
|
startListening(generation,{preserveDisplay:true});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// The utterance is dispatched under the CURRENT response epoch: when a
|
||||||
|
// live-capture barge just cancelled a model turn, this transcript is
|
||||||
|
// the interruption that replaces it.
|
||||||
transcribeStreamingOrFallback(
|
transcribeStreamingOrFallback(
|
||||||
new Blob(chunks,{type:recordedMime||'audio/webm'}),
|
new Blob(chunks,{type:recordedMime||'audio/webm'}),
|
||||||
token,
|
generation,
|
||||||
session,
|
session,
|
||||||
!(reusedCapture&&reusedCapture.requireStreamingLookback)
|
!(reusedCapture&&reusedCapture.requireStreamingLookback),
|
||||||
|
utteranceTurnId
|
||||||
);
|
);
|
||||||
|
// Re-enter capture immediately — never wait for transcription or the
|
||||||
|
// response. The display may show transcribing/thinking while the next
|
||||||
|
// capture turn is already live underneath.
|
||||||
|
startListening(generation,{preserveDisplay:true});
|
||||||
};
|
};
|
||||||
// Ask the browser for one finalized container at stop. Android Chromium
|
// Ask the browser for one finalized container at stop. Android Chromium
|
||||||
// can emit timeslice fragments without a reusable EBML initialization
|
// can emit timeslice fragments without a reusable EBML initialization
|
||||||
@ -1054,7 +1199,7 @@
|
|||||||
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1100',10)||1100);
|
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1100',10)||1100);
|
||||||
const speculateMs=Math.min(silenceMs-250,Math.max(450,Math.round(silenceMs*0.55)));
|
const speculateMs=Math.min(silenceMs-250,Math.max(450,Math.round(silenceMs*0.55)));
|
||||||
vadTimer=window.setInterval(function(){
|
vadTimer=window.setInterval(function(){
|
||||||
if(!active||token!==generation||!recorder||recorder.state==='inactive') return;
|
if(!active||captureToken!==captureGeneration||!recorder||recorder.state==='inactive') return;
|
||||||
analyser.getByteTimeDomainData(samples);
|
analyser.getByteTimeDomainData(samples);
|
||||||
let energy=0;
|
let energy=0;
|
||||||
for(let index=0;index<samples.length;index+=1){
|
for(let index=0;index<samples.length;index+=1){
|
||||||
@ -1064,13 +1209,39 @@
|
|||||||
const rms=Math.sqrt(energy/samples.length);
|
const rms=Math.sqrt(energy/samples.length);
|
||||||
updateInputLevel(rms);
|
updateInputLevel(rms);
|
||||||
const now=Date.now();
|
const now=Date.now();
|
||||||
|
// Continuous capture stays live while Hermes thinks and speaks, so the
|
||||||
|
// same VAD that endpoints utterances is also the barge-in detector.
|
||||||
|
// While our own audio plays the microphone hears the speakers: without
|
||||||
|
// AEC nothing counts as speech at all, and with AEC a post-playback
|
||||||
|
// arming delay plus the sustained-frame requirement below mirror the
|
||||||
|
// conservative energy monitor this path supersedes.
|
||||||
|
const playbackLive=indicator.classList.contains('is-playing')||!!currentAudio||!!(playbackSession&&playbackSession.node)||!!(thinkingCue&&thinkingCue.node);
|
||||||
|
if(playbackLive&&!playbackWasLive){
|
||||||
|
bargeArmAt=now+(state==='thinking'?150:450);
|
||||||
|
}else if(!playbackLive&&playbackWasLive){
|
||||||
|
bargeArmAt=now+100;
|
||||||
|
}
|
||||||
|
playbackWasLive=playbackLive;
|
||||||
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
||||||
const voiceNow=rms>speechThreshold;
|
const voiceNow=rms>speechThreshold&&(!playbackLive||(captureAec&&now>=bargeArmAt));
|
||||||
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(rms*0.06);}
|
||||||
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
||||||
if(!heardSpeech&&voiceFrames>=3){
|
if(playbackLive){
|
||||||
|
if(voiceFrames>=BARGE_DUCK_FRAMES&&!ducked){ducked=true;setPlaybackDucked(true);}
|
||||||
|
if(!voiceFrames&&ducked){ducked=false;setPlaybackDucked(false);}
|
||||||
|
}else if(ducked){
|
||||||
|
ducked=false;
|
||||||
|
setPlaybackDucked(false);
|
||||||
|
}
|
||||||
|
// Speech onset. While playback runs an echo-safe barge needs the same
|
||||||
|
// sustained evidence (BARGE_TRIGGER_FRAMES) as the old energy monitor.
|
||||||
|
const onsetFrames=playbackLive?BARGE_TRIGGER_FRAMES:3;
|
||||||
|
if(!heardSpeech&&voiceFrames>=onsetFrames){
|
||||||
heardSpeech=true;
|
heardSpeech=true;
|
||||||
lastSpeech=now;
|
lastSpeech=now;
|
||||||
|
// The user started a new utterance while Hermes was mid-response:
|
||||||
|
// cancel the response side only; this capture keeps running.
|
||||||
|
if(state==='thinking'||state==='speaking') bargeFromLiveCapture();
|
||||||
}else if(heardSpeech&&voiceNow){
|
}else if(heardSpeech&&voiceNow){
|
||||||
lastSpeech=now;
|
lastSpeech=now;
|
||||||
if(speculative&&streamingStt){streamingStt.resume();speculative=false;}
|
if(speculative&&streamingStt){streamingStt.resume();speculative=false;}
|
||||||
@ -1092,7 +1263,10 @@
|
|||||||
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
||||||
reusedCapture.handoffMonitor=null;
|
reusedCapture.handoffMonitor=null;
|
||||||
}
|
}
|
||||||
if(!active||token!==generation) return;
|
if(!active||captureToken!==captureGeneration) return;
|
||||||
|
// This capture turn is dead. A fatal microphone error ends the session
|
||||||
|
// and releases the retained microphone via showUnavailable().
|
||||||
|
captureActive=false;
|
||||||
const message=errorMessage(error,'Microphone permission is required');
|
const message=errorMessage(error,'Microphone permission is required');
|
||||||
showUnavailable(message);
|
showUnavailable(message);
|
||||||
toast(message);
|
toast(message);
|
||||||
@ -1232,8 +1406,17 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ttsSpeed(){
|
||||||
|
// Hands-free speech-rate preference. Piper accepts 0.5-2.0 (length_scale
|
||||||
|
// 1/speed, no pitch shift); hands-free defaults to a slightly brisk 1.15,
|
||||||
|
// seeded once at initialize() and user-tunable via localStorage.
|
||||||
|
const stored=parseFloat(localStorage.getItem('hermes-voice-tts-speed')||'');
|
||||||
|
if(!Number.isFinite(stored)) return TTS_SPEED_DEFAULT;
|
||||||
|
return Math.min(TTS_SPEED_MAX,Math.max(TTS_SPEED_MIN,stored));
|
||||||
|
}
|
||||||
|
|
||||||
function ttsRequest(chunk,language,turnId){
|
function ttsRequest(chunk,language,turnId){
|
||||||
const request={text:chunk,engine:'atlas',turn_id:turnId};
|
const request={text:chunk,engine:'atlas',turn_id:turnId,speed:ttsSpeed()};
|
||||||
if(language) request.language=language;
|
if(language) request.language=language;
|
||||||
return request;
|
return request;
|
||||||
}
|
}
|
||||||
@ -1624,6 +1807,11 @@
|
|||||||
|
|
||||||
function pumpAssistantResponse(token,isFinal){
|
function pumpAssistantResponse(token,isFinal){
|
||||||
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
||||||
|
if(isFinal){
|
||||||
|
// The completion callback marks a normally completed response: any
|
||||||
|
// interrupted-thought stitch from an earlier barge is now stale.
|
||||||
|
pendingStitch=null;
|
||||||
|
}
|
||||||
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
||||||
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
||||||
thinkingSession=null;
|
thinkingSession=null;
|
||||||
@ -1758,6 +1946,11 @@
|
|||||||
localStorage.setItem('hermes-atlas-voice-latency-v2','1');
|
localStorage.setItem('hermes-atlas-voice-latency-v2','1');
|
||||||
localStorage.setItem('hermes-voice-silence-ms','1100');
|
localStorage.setItem('hermes-voice-silence-ms','1100');
|
||||||
}
|
}
|
||||||
|
// Seed the hands-free speech-rate default once; later user changes to
|
||||||
|
// hermes-voice-tts-speed (clamped to 0.5-2.0) are respected as-is.
|
||||||
|
if(localStorage.getItem('hermes-voice-tts-speed')===null){
|
||||||
|
localStorage.setItem('hermes-voice-tts-speed',String(TTS_SPEED_DEFAULT));
|
||||||
|
}
|
||||||
const selector=document.getElementById('settingsTtsEngine');
|
const selector=document.getElementById('settingsTtsEngine');
|
||||||
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
||||||
const option=document.createElement('option');
|
const option=document.createElement('option');
|
||||||
|
|||||||
@ -158,7 +158,10 @@ def test_visual_slice_preserves_private_voice_request_and_capture_contract():
|
|||||||
assert script.count("navigator.mediaDevices.getUserMedia(") == 1
|
assert script.count("navigator.mediaDevices.getUserMedia(") == 1
|
||||||
assert "form.append('file',new File([blob],'voice-input.'+ext" in script
|
assert "form.append('file',new File([blob],'voice-input.'+ext" in script
|
||||||
assert "fetch('/api/transcribe',{method:'POST',body:form})" in script
|
assert "fetch('/api/transcribe',{method:'POST',body:form})" in script
|
||||||
assert "const request={text:chunk,engine:'atlas',turn_id:turnId}" in script
|
assert (
|
||||||
|
"const request={text:chunk,engine:'atlas',turn_id:turnId,speed:ttsSpeed()}"
|
||||||
|
in script
|
||||||
|
)
|
||||||
assert "if(language) request.language=language" in script
|
assert "if(language) request.language=language" in script
|
||||||
assert "speakResponse(generation)" in script
|
assert "speakResponse(generation)" in script
|
||||||
assert "window._voiceModeImmediateSend" in script
|
assert "window._voiceModeImmediateSend" in script
|
||||||
@ -200,4 +203,4 @@ def test_release_candidate_keeps_finalized_webm_as_quality_fallback():
|
|||||||
assert "recorder.start(250)" not in script
|
assert "recorder.start(250)" not in script
|
||||||
assert "new Blob(chunks,{type:recordedMime||'audio/webm'})" in script
|
assert "new Blob(chunks,{type:recordedMime||'audio/webm'})" in script
|
||||||
assert "transcribeStreamingOrFallback" in script
|
assert "transcribeStreamingOrFallback" in script
|
||||||
assert "transcribe(blob,token)" in script
|
assert "transcribe(blob,token,turnId)" in script
|
||||||
|
|||||||
@ -624,7 +624,7 @@ def test_voice_mode_drops_hostile_language_values(voice_probe):
|
|||||||
|
|
||||||
def test_voice_mode_never_sends_a_voice_field(voice_probe):
|
def test_voice_mode_never_sends_a_voice_field(voice_probe):
|
||||||
for request in voice_probe["voice_field_is_never_sent"]["tts"]:
|
for request in voice_probe["voice_field_is_never_sent"]["tts"]:
|
||||||
assert set(request) <= {"text", "engine", "language", "turn_id"}
|
assert set(request) <= {"text", "engine", "language", "turn_id", "speed"}
|
||||||
assert "voice" not in request
|
assert "voice" not in request
|
||||||
|
|
||||||
|
|
||||||
@ -797,10 +797,20 @@ def test_streaming_tts_payload_is_narrow_and_turn_bound(patched_webui):
|
|||||||
assert payload == {
|
assert payload == {
|
||||||
"model": "piper",
|
"model": "piper",
|
||||||
"input": "A safe sentence.",
|
"input": "A safe sentence.",
|
||||||
"speed": 1.0,
|
"speed": 2.0,
|
||||||
"language": "ru",
|
"language": "ru",
|
||||||
"turn_id": "voice-turn-7",
|
"turn_id": "voice-turn-7",
|
||||||
}
|
}
|
||||||
|
assert patched_webui.routes._atlas_tts_stream_payload(
|
||||||
|
{"text": "A safe sentence.", "speed": 1.15}
|
||||||
|
)["speed"] == 1.15
|
||||||
|
assert patched_webui.routes._atlas_tts_stream_payload(
|
||||||
|
{"text": "A safe sentence."}
|
||||||
|
)["speed"] == 1.0
|
||||||
|
for hostile in ("2", True, None, [1.5], {"speed": 1.5}, float("nan")):
|
||||||
|
assert patched_webui.routes._atlas_tts_stream_payload(
|
||||||
|
{"text": "A safe sentence.", "speed": hostile}
|
||||||
|
)["speed"] == 1.0
|
||||||
|
|
||||||
|
|
||||||
def test_streaming_tts_payload_forwards_only_allowlisted_localized_cues(patched_webui):
|
def test_streaming_tts_payload_forwards_only_allowlisted_localized_cues(patched_webui):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user