2026-08-21 06:43:18 +00:00
// Natural turn-taking for chat.bstein.dev using the private Jetsons.
2026-08-10 00:42:37 -03:00
( function ( ) {
'use strict' ;
const modeBtn = document . getElementById ( 'btnVoiceMode' ) ;
const bar = document . getElementById ( 'voiceModeBar' ) ;
const indicator = document . getElementById ( 'voiceModeIndicator' ) ;
const label = document . getElementById ( 'voiceModeLabel' ) ;
const composer = document . getElementById ( 'msg' ) ;
if ( ! modeBtn || ! bar || ! indicator || ! label || ! composer || ! navigator . mediaDevices || ! window . MediaRecorder ) return ;
let ready = false ;
let active = false ;
let state = 'idle' ;
let generation = 0 ;
2026-08-23 18:29:02 -03:00
let turnSequence = 0 ;
2026-08-10 00:42:37 -03:00
let recorder = null ;
let stream = null ;
let audioContext = null ;
2026-08-23 18:29:02 -03:00
let captureNode = null ;
2026-08-10 00:42:37 -03:00
let vadTimer = null ;
2026-08-23 18:29:02 -03:00
let responsePollTimer = null ;
2026-08-10 00:42:37 -03:00
let currentAudio = null ;
2026-08-23 18:29:02 -03:00
let playbackSession = null ;
2026-08-23 22:13:52 -03:00
let bargeMonitor = null ;
let bargeCancelPromise = null ;
let suppressAutoRead = false ;
let thinkingCue = null ;
2026-08-23 18:29:02 -03:00
let streamingStt = null ;
let captureTurnId = '' ;
let thinkingTurnId = '' ;
2026-08-10 00:42:37 -03:00
let thinkingSession = null ;
2026-08-23 18:29:02 -03:00
let assistantBaseline = null ;
let speechTurn = null ;
2026-08-20 23:13:51 +00:00
let errorTimer = null ;
let visualInputLevel = 0 ;
2026-08-23 22:13:52 -03:00
let streamingCapability = { tts : null , stt : null , preflight : null } ;
let voicePreflight = null ;
const voiceTabNonce = ( function ( ) {
try {
const bytes = new Uint8Array ( 16 ) ;
window . crypto . getRandomValues ( bytes ) ;
return Array . from ( bytes , function ( value ) { return value . toString ( 16 ) . padStart ( 2 , '0' ) ; } ) . join ( '' ) ;
} catch ( _ ) { return '' ; }
} ) ( ) ;
2026-08-20 23:13:51 +00:00
const reducedMotion = window . matchMedia ? window . matchMedia ( '(prefers-reduced-motion: reduce)' ) : { matches : false } ;
const ERROR _VISIBLE _MS = 3200 ;
2026-08-23 18:29:02 -03:00
const STREAMING _CAPABILITY _URL = '/api/voice/streaming/capability' ;
const TTS _STREAM _URL = '/api/tts/stream' ;
const STT _STREAM _PATH = '/api/transcribe/stream' ;
2026-08-23 22:13:52 -03:00
const VOICE _PREFLIGHT _URL = '/api/voice/route-preflight' ;
const VOICE _PREFLIGHT _DEBOUNCE _MS = 200 ;
const VOICE _PREFLIGHT _DEADLINE _MS = 1100 ;
const VOICE _PREFLIGHT _TIERS = [ 'fast' , 'balanced' , 'deep' , 'maximum' ] ;
2026-08-23 18:29:02 -03:00
const WORKLET _URL = 'static/atlas-voice-worklet.js' ;
2026-08-23 22:13:52 -03:00
const STT _MAX _QUEUED _BYTES = 1048576 ;
const STT _MAX _ARCHIVE _BYTES = 2880000 ;
const BARGE _LOOKBACK _MS = 900 ;
const BARGE _DUCK _FRAMES = 2 ;
const BARGE _TRIGGER _FRAMES = 4 ;
const THINKING _CUE _FIRST _MS = 1900 ;
const THINKING _CUE _INTERVAL _MS = 6500 ;
const THINKING _CUE _POOLS = {
en : [ { id : 'thinking' , text : "I'm thinking." } , { id : 'let_me_think' , text : 'Let me think.' } , { id : 'still_working' , text : 'Still working on that.' } , { id : 'one_more_moment' , text : 'One more moment.' } ] ,
ru : [ { id : 'thinking' , text : 'Я думаю.' } , { id : 'let_me_think' , text : 'Дайте подумать.' } , { id : 'still_working' , text : 'Я всё ещё думаю над этим.' } , { id : 'one_more_moment' , text : 'Ещё мгновение.' } ] ,
es : [ { id : 'thinking' , text : 'Estoy pensando.' } , { id : 'let_me_think' , text : 'Déjame pensar.' } , { id : 'still_working' , text : 'Sigo pensando en eso.' } , { id : 'one_more_moment' , text : 'Un momento más.' } ] ,
} ;
2026-08-20 23:13:51 +00:00
const STATE _LABELS = {
listening : 'Listening' ,
transcribing : 'Transcribing…' ,
thinking : 'Thinking…' ,
speaking : 'Speaking' ,
error : 'Voice unavailable' ,
idle : '' ,
} ;
2026-08-21 11:52:49 +00:00
// The only language signal this file trusts is the one the private Whisper
// service returned for the audio of the turn currently being answered. It is
// bound to that turn's generation token and consumed exactly once.
let sttLanguage = '' ;
let sttLanguageToken = - 1 ;
const TTS _LANGUAGES = [ 'en' , 'ru' , 'es' ] ;
2026-08-10 00:42:37 -03:00
const originalAutoRead = window . autoReadLastAssistant ;
const originalApplyPreference = window . _applyVoiceModePref ;
2026-08-21 11:52:49 +00:00
function normalizeSttLanguage ( value ) {
if ( typeof value !== 'string' ) return '' ;
const code = value . trim ( ) . toLowerCase ( ) ;
return TTS _LANGUAGES . indexOf ( code ) >= 0 ? code : '' ;
}
function clearSttLanguage ( ) {
sttLanguage = '' ;
sttLanguageToken = - 1 ;
}
2026-08-23 18:29:02 -03:00
function rememberSttLanguage ( language , token ) {
2026-08-21 11:52:49 +00:00
sttLanguage = language || '' ;
sttLanguageToken = sttLanguage ? token : - 1 ;
}
function takeSttLanguage ( token ) {
const language = sttLanguageToken === token ? sttLanguage : '' ;
clearSttLanguage ( ) ;
return language ;
}
2026-08-10 00:42:37 -03:00
function toast ( message ) {
if ( typeof window . showToast === 'function' ) window . showToast ( message , 3000 ) ;
}
2026-08-23 18:29:02 -03:00
function setState ( next , customLabel ) {
2026-08-10 00:42:37 -03:00
state = next ;
indicator . className = 'voice-mode-indicator ' + next ;
2026-08-20 23:13:51 +00:00
bar . dataset . voiceState = next ;
bar . setAttribute ( 'aria-busy' , next === 'transcribing' || next === 'thinking' ? 'true' : 'false' ) ;
label . textContent = customLabel || STATE _LABELS [ next ] || '' ;
bar . style . display = ( active && next !== 'idle' ) || next === 'error' ? '' : 'none' ;
resetInputLevel ( ) ;
}
function resetInputLevel ( ) {
visualInputLevel = 0 ;
indicator . style . setProperty ( '--voice-ripple-scale' , '1.035' ) ;
indicator . style . setProperty ( '--voice-ripple-opacity' , '0.3' ) ;
}
function updateInputLevel ( rms ) {
if ( reducedMotion . matches || state !== 'listening' ) return ;
const target = Math . max ( 0 , Math . min ( 1 , ( rms - 0.01 ) / 0.18 ) ) ;
visualInputLevel = ( visualInputLevel * 0.72 ) + ( target * 0.28 ) ;
indicator . style . setProperty ( '--voice-ripple-scale' , ( 1.035 + ( visualInputLevel * 0.16 ) ) . toFixed ( 3 ) ) ;
indicator . style . setProperty ( '--voice-ripple-opacity' , ( 0.26 + ( visualInputLevel * 0.48 ) ) . toFixed ( 3 ) ) ;
}
function clearErrorTimer ( ) {
if ( errorTimer ) { window . clearTimeout ( errorTimer ) ; errorTimer = null ; }
}
2026-08-23 18:29:02 -03:00
function errorMessage ( error , fallback ) {
2026-08-20 23:13:51 +00:00
return String ( ( error && error . message ) || fallback || 'Voice unavailable' ) . trim ( ) ;
}
2026-08-23 22:13:52 -03:00
function createAbortController ( ) {
const Controller = window . AbortController || ( typeof AbortController !== 'undefined' ? AbortController : null ) ;
return Controller ? new Controller ( ) : { signal : undefined , abort : function ( ) { } } ;
}
function cancelledError ( message ) {
const error = new Error ( message || 'Voice turn cancelled' ) ;
error . name = 'AbortError' ;
return error ;
}
2026-08-23 18:29:02 -03:00
function cancelSpeechTurn ( ) {
if ( ! speechTurn ) return ;
speechTurn . cancelled = true ;
speechTurn . final = true ;
while ( speechTurn . waiters . length ) speechTurn . waiters . shift ( ) ( null ) ;
speechTurn = null ;
}
function cancelStreamingStt ( ) {
if ( ! streamingStt ) return ;
streamingStt . cancel ( ) ;
streamingStt = null ;
}
2026-08-23 22:13:52 -03:00
function cancelVoicePreflight ( turnId ) {
const preflight = voicePreflight ;
if ( ! preflight || ( turnId && preflight . turnId !== turnId ) ) return ;
voicePreflight = null ;
if ( preflight . timer ) { window . clearTimeout ( preflight . timer ) ; preflight . timer = null ; }
if ( preflight . deadline ) { window . clearTimeout ( preflight . deadline ) ; preflight . deadline = null ; }
if ( preflight . controller ) { preflight . controller . abort ( ) ; preflight . controller = null ; }
}
function scheduleVoicePreflight ( turnId , revision , transcript ) {
const capability = streamingCapability . preflight ;
const stable = String ( transcript || '' ) . replace ( /\s+/g , ' ' ) . trim ( ) ;
if ( ! capability || ! voiceTabNonce || ! active || captureTurnId !== turnId || state !== 'listening' ) return ;
if ( ! Number . isInteger ( revision ) || revision < 1 || stable . length < 12 || stable . length > 512 ) return ;
if ( voicePreflight && voicePreflight . turnId === turnId && voicePreflight . transcript === stable ) return ;
cancelVoicePreflight ( ) ;
const preflight = { turnId : turnId , revision : revision , transcript : stable , timer : null , deadline : null , controller : null } ;
voicePreflight = preflight ;
preflight . timer = window . setTimeout ( async function ( ) {
preflight . timer = null ;
if ( voicePreflight !== preflight || ! active || captureTurnId !== turnId || state !== 'listening' ) return ;
const controller = createAbortController ( ) ;
preflight . controller = controller ;
preflight . deadline = window . setTimeout ( function ( ) { controller . abort ( ) ; } , VOICE _PREFLIGHT _DEADLINE _MS ) ;
try {
const response = await fetch ( VOICE _PREFLIGHT _URL , {
method : 'POST' ,
credentials : 'same-origin' ,
cache : 'no-store' ,
headers : { 'Content-Type' : 'application/json' } ,
body : JSON . stringify ( { turn _id : turnId , revision : revision , transcript : stable } ) ,
signal : controller . signal ,
} ) ;
if ( ! response . ok ) return ;
const advisory = await response . json ( ) ;
const tier = advisory && advisory . tier ;
if (
voicePreflight !== preflight || ! active || captureTurnId !== turnId || state !== 'listening' ||
advisory . turn _id !== turnId || advisory . revision !== revision || advisory . advisory !== true ||
VOICE _PREFLIGHT _TIERS . indexOf ( tier ) < 0 || advisory . target !== 'atlas/auto/' + tier
) return ;
// This is intentionally feedback-only. The completed transcript still
// takes the normal Switchyard path and remains the sole routing input.
const preview = stable . length > 60 ? stable . slice ( 0 , 57 ) + '…' : stable ;
label . textContent = 'Listening · ' + preview + ' · ' + tier ;
} catch ( _ ) {
// A partial can be superseded at any time; advisory failure is silent.
} finally {
if ( preflight . deadline ) { window . clearTimeout ( preflight . deadline ) ; preflight . deadline = null ; }
preflight . controller = null ;
}
} , VOICE _PREFLIGHT _DEBOUNCE _MS ) ;
}
function cancelThinkingCues ( ) {
const cue = thinkingCue ;
thinkingCue = null ;
if ( ! cue ) return ;
cue . cancelled = true ;
if ( cue . timer ) { window . clearTimeout ( cue . timer ) ; cue . timer = null ; }
if ( cue . controller ) { cue . controller . abort ( ) ; cue . controller = null ; }
if ( cue . audioWake ) { const wake = cue . audioWake ; cue . audioWake = null ; wake ( ) ; }
if ( cue . node ) { try { cue . node . port . postMessage ( { type : 'cancel' } ) ; cue . node . disconnect ( ) ; } catch ( _ ) { } cue . node = null ; }
if ( cue . context ) { try { cue . context . close ( ) ; } catch ( _ ) { } cue . context = null ; }
cue . gain = null ;
indicator . classList . remove ( 'is-playing' ) ;
indicator . classList . remove ( 'is-ducked' ) ;
}
function clearBargeCancellation ( ) {
const cancellation = bargeCancelPromise ;
bargeCancelPromise = null ;
if ( cancellation && cancellation . controller ) cancellation . controller . abort ( ) ;
}
function microphoneConstraints ( ) {
const constraints = { echoCancellation : true , noiseSuppression : true , autoGainControl : true } ;
const supported = navigator . mediaDevices . getSupportedConstraints ? navigator . mediaDevices . getSupportedConstraints ( ) : { } ;
if ( supported . voiceIsolation ) constraints . voiceIsolation = true ;
return constraints ;
}
function acquireMicrophone ( ) {
return navigator . mediaDevices . getUserMedia ( { audio : microphoneConstraints ( ) } ) ;
}
function captureAecIsUsable ( capture ) {
try {
const tracks = capture && capture . getAudioTracks ? capture . getAudioTracks ( ) : [ ] ;
const settings = tracks . length && tracks [ 0 ] . getSettings ? tracks [ 0 ] . getSettings ( ) : { } ;
return settings . echoCancellation !== false ;
} catch ( _ ) {
return true ;
}
}
window . _atlasCaptureAecIsUsable = captureAecIsUsable ;
function setPlaybackDucked ( ducked ) {
const session = playbackSession ;
if ( session && session . gain && session . context ) {
const gain = ducked ? 0.16 : 1 ;
try { session . gain . gain . setTargetAtTime ( gain , session . context . currentTime , 0.018 ) ; } catch ( _ ) { session . gain . gain . value = gain ; }
}
if ( currentAudio ) currentAudio . volume = ducked ? 0.16 : 1 ;
const cue = thinkingCue ;
if ( cue && cue . gain && cue . context ) {
const gain = ducked ? 0.16 : 1 ;
try { cue . gain . gain . setTargetAtTime ( gain , cue . context . currentTime , 0.018 ) ; } catch ( _ ) { cue . gain . gain . value = gain ; }
}
if ( ducked ) indicator . classList . add ( 'is-ducked' ) ; else indicator . classList . remove ( 'is-ducked' ) ;
}
function disposeBargeResources ( monitor , keepCapture ) {
if ( ! monitor ) return ;
monitor . cancelled = true ;
if ( monitor . timer ) { window . clearInterval ( monitor . timer ) ; monitor . timer = null ; }
try { if ( monitor . node ) monitor . node . disconnect ( ) ; } catch ( _ ) { }
try { if ( monitor . source ) monitor . source . disconnect ( ) ; } catch ( _ ) { }
try { if ( monitor . silentGain ) monitor . silentGain . disconnect ( ) ; } catch ( _ ) { }
if ( ! keepCapture ) {
if ( monitor . stream ) monitor . stream . getTracks ( ) . forEach ( function ( track ) { track . stop ( ) ; } ) ;
if ( monitor . context ) { try { monitor . context . close ( ) ; } catch ( _ ) { } }
}
if ( ! keepCapture ) setPlaybackDucked ( false ) ;
}
function stopBargeMonitor ( ) {
const monitor = bargeMonitor ;
bargeMonitor = null ;
disposeBargeResources ( monitor , false ) ;
}
2026-08-20 23:13:51 +00:00
function showUnavailable ( message ) {
generation += 1 ;
active = false ;
thinkingSession = null ;
2026-08-23 18:29:02 -03:00
thinkingTurnId = '' ;
2026-08-23 22:13:52 -03:00
suppressAutoRead = false ;
clearBargeCancellation ( ) ;
2026-08-23 18:29:02 -03:00
stopResponseObserver ( ) ;
2026-08-23 22:13:52 -03:00
cancelThinkingCues ( ) ;
2026-08-23 18:29:02 -03:00
cancelSpeechTurn ( ) ;
2026-08-20 23:13:51 +00:00
stopCapture ( ) ;
stopPlayback ( ) ;
modeBtn . classList . remove ( 'active' ) ;
setState ( 'error' , message ) ;
clearErrorTimer ( ) ;
errorTimer = window . setTimeout ( function ( ) {
errorTimer = null ;
if ( ! active && state === 'error' && indicator . classList . contains ( 'error' ) ) setState ( 'idle' ) ;
} , ERROR _VISIBLE _MS ) ;
2026-08-10 00:42:37 -03:00
}
function stopCapture ( ) {
2026-08-23 22:13:52 -03:00
cancelVoicePreflight ( captureTurnId ) ;
stopBargeMonitor ( ) ;
2026-08-10 00:42:37 -03:00
if ( vadTimer ) { clearInterval ( vadTimer ) ; vadTimer = null ; }
if ( recorder && recorder . state !== 'inactive' ) {
try { recorder . stop ( ) ; } catch ( _ ) { }
}
recorder = null ;
2026-08-23 18:29:02 -03:00
captureNode = null ;
cancelStreamingStt ( ) ;
2026-08-10 00:42:37 -03:00
if ( stream ) { stream . getTracks ( ) . forEach ( function ( track ) { track . stop ( ) ; } ) ; stream = null ; }
if ( audioContext ) { try { audioContext . close ( ) ; } catch ( _ ) { } audioContext = null ; }
}
function stopPlayback ( ) {
2026-08-23 18:29:02 -03:00
if ( currentAudio ) {
try { currentAudio . pause ( ) ; currentAudio . currentTime = 0 ; } catch ( _ ) { }
currentAudio = null ;
}
if ( playbackSession ) {
playbackSession . cancelled = true ;
if ( playbackSession . blobWake ) { playbackSession . blobWake ( ) ; playbackSession . blobWake = null ; }
playbackSession . controllers . forEach ( function ( controller ) { controller . abort ( ) ; } ) ;
if ( playbackSession . lowWaterWake ) {
playbackSession . lowWaterWake ( ) ;
playbackSession . lowWaterWake = null ;
}
if ( playbackSession . drainWake ) { playbackSession . drainWake ( ) ; playbackSession . drainWake = null ; }
if ( playbackSession . node ) {
try { playbackSession . node . port . postMessage ( { type : 'cancel' } ) ; playbackSession . node . disconnect ( ) ; } catch ( _ ) { }
}
if ( playbackSession . context ) { try { playbackSession . context . close ( ) ; } catch ( _ ) { } }
playbackSession = null ;
}
2026-08-20 23:13:51 +00:00
indicator . classList . remove ( 'is-playing' ) ;
2026-08-23 22:13:52 -03:00
indicator . classList . remove ( 'is-ducked' ) ;
2026-08-10 00:42:37 -03:00
}
2026-08-23 18:29:02 -03:00
function stopResponseObserver ( ) {
if ( responsePollTimer ) { clearInterval ( responsePollTimer ) ; responsePollTimer = null ; }
}
2026-08-10 00:42:37 -03:00
function deactivate ( showMessage ) {
generation += 1 ;
active = false ;
thinkingSession = null ;
2026-08-23 18:29:02 -03:00
thinkingTurnId = '' ;
2026-08-20 23:13:51 +00:00
clearErrorTimer ( ) ;
2026-08-21 11:52:49 +00:00
clearSttLanguage ( ) ;
2026-08-23 22:13:52 -03:00
suppressAutoRead = false ;
clearBargeCancellation ( ) ;
2026-08-23 18:29:02 -03:00
stopResponseObserver ( ) ;
2026-08-23 22:13:52 -03:00
cancelThinkingCues ( ) ;
2026-08-23 18:29:02 -03:00
cancelSpeechTurn ( ) ;
2026-08-10 00:42:37 -03:00
stopCapture ( ) ;
stopPlayback ( ) ;
modeBtn . classList . remove ( 'active' ) ;
2026-08-20 23:13:51 +00:00
setState ( 'idle' ) ;
2026-08-10 00:42:37 -03:00
if ( showMessage ) toast ( 'Hands-free voice mode off' ) ;
}
2026-08-23 18:29:02 -03:00
function restartSoon ( token , delay ) {
2026-08-10 00:42:37 -03:00
window . setTimeout ( function ( ) {
if ( active && token === generation ) startListening ( token ) ;
} , delay || 500 ) ;
}
2026-08-23 18:29:02 -03:00
function assistantRows ( ) {
return document . querySelectorAll ( '.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]' ) ;
}
function readAssistantRow ( row ) {
if ( ! row ) return '' ;
if ( row . dataset && typeof row . dataset . rawText === 'string' ) return row . dataset . rawText ;
return typeof row . textContent === 'string' ? row . textContent : '' ;
}
function rememberAssistantBaseline ( ) {
const rows = assistantRows ( ) ;
const row = rows . length ? rows [ rows . length - 1 ] : null ;
assistantBaseline = { row : row , text : readAssistantRow ( row ) , count : rows . length } ;
}
2026-08-23 22:13:52 -03:00
async function settleBargeCancellation ( token ) {
const cancellation = bargeCancelPromise ;
if ( ! cancellation ) return true ;
try { await cancellation . promise ; } catch ( _ ) { }
const deadline = Date . now ( ) + 10000 ;
while ( active && token === generation && Date . now ( ) < deadline ) {
const sessionId = ( typeof S !== 'undefined' && S . session ) ? S . session . session _id : '' ;
const activeStreamId = typeof S !== 'undefined' ? String ( S . activeStreamId || '' ) : '' ;
const busy = typeof S !== 'undefined' && ! ! S . busy ;
if ( sessionId !== cancellation . sessionId ) break ;
if ( ! busy && ( ! cancellation . streamId || activeStreamId !== cancellation . streamId ) ) {
if ( bargeCancelPromise === cancellation ) bargeCancelPromise = null ;
return true ;
}
await new Promise ( function ( resolve ) { window . setTimeout ( resolve , 50 ) ; } ) ;
}
if ( bargeCancelPromise === cancellation ) bargeCancelPromise = null ;
return false ;
}
async function sendTranscript ( transcript , token , language ) {
2026-08-10 00:42:37 -03:00
if ( ! active || token !== generation ) return ;
2026-08-23 22:13:52 -03:00
cancelVoicePreflight ( captureTurnId ) ;
2026-08-10 00:42:37 -03:00
const text = String ( transcript || '' ) . trim ( ) ;
2026-08-21 11:52:49 +00:00
if ( ! text ) { clearSttLanguage ( ) ; restartSoon ( token , 350 ) ; return ; }
2026-08-10 00:42:37 -03:00
composer . value = text ;
if ( typeof window . autoResize === 'function' ) window . autoResize ( ) ;
2026-08-23 22:13:52 -03:00
setState ( 'thinking' ) ;
startBargeMonitor ( token ) ;
if ( ! bargeCancelPromise && typeof S !== 'undefined' && ( S . busy || S . activeStreamId ) ) {
suppressAutoRead = true ;
bargeCancelPromise = cancelActiveModelTurn ( ) ;
}
const cancellationSettled = await settleBargeCancellation ( token ) ;
if ( ! active || token !== generation ) return ;
if ( ! cancellationSettled ) {
toast ( 'The previous response did not stop. Please repeat your interruption.' ) ;
restartSoon ( token , 250 ) ;
return ;
}
2026-08-10 00:42:37 -03:00
thinkingSession = ( typeof S !== 'undefined' && S . session ) ? S . session . session _id : null ;
2026-08-23 18:29:02 -03:00
thinkingTurnId = captureTurnId || String ( token ) + '-' + String ( ++ turnSequence ) ;
rememberAssistantBaseline ( ) ;
2026-08-21 11:52:49 +00:00
rememberSttLanguage ( language , token ) ;
2026-08-23 22:13:52 -03:00
if ( typeof window . send === 'function' ) {
window . send ( ) ;
suppressAutoRead = false ;
startResponseObserver ( token ) ;
scheduleThinkingCues ( token , language , thinkingTurnId ) ;
}
2026-08-10 00:42:37 -03:00
}
2026-08-21 11:12:15 +00:00
function audioExtension ( mimeType ) {
const normalized = String ( mimeType || '' ) . toLowerCase ( ) ;
2026-08-23 22:13:52 -03:00
if ( normalized . indexOf ( 'wav' ) >= 0 ) return 'wav' ;
2026-08-21 11:12:15 +00:00
if ( normalized . indexOf ( 'ogg' ) >= 0 ) return 'ogg' ;
if ( normalized . indexOf ( 'mp4' ) >= 0 ) return 'mp4' ;
return 'webm' ;
}
2026-08-23 18:29:02 -03:00
async function transcribe ( blob , token ) {
2026-08-10 00:42:37 -03:00
if ( ! active || token !== generation ) return ;
2026-08-20 23:13:51 +00:00
setState ( 'transcribing' ) ;
2026-08-21 11:12:15 +00:00
const ext = audioExtension ( blob . type ) ;
2026-08-10 00:42:37 -03:00
const form = new FormData ( ) ;
form . append ( 'file' , new File ( [ blob ] , 'voice-input.' + ext , { type : blob . type || 'audio/' + ext } ) ) ;
try {
const response = await fetch ( '/api/transcribe' , { method : 'POST' , body : form } ) ;
const payload = await response . json ( ) . catch ( function ( ) { return { } ; } ) ;
if ( ! response . ok ) throw new Error ( payload . error || ( 'Whisper request failed: ' + response . status ) ) ;
2026-08-21 11:52:49 +00:00
sendTranscript ( payload . transcript , token , normalizeSttLanguage ( payload . language ) ) ;
2026-08-10 00:42:37 -03:00
} catch ( error ) {
if ( ! active || token !== generation ) return ;
2026-08-20 23:13:51 +00:00
const message = errorMessage ( error , 'Private Whisper is unavailable' ) ;
showUnavailable ( message ) ;
toast ( message ) ;
2026-08-10 00:42:37 -03:00
// If the browser supplies its own recognizer, hand control back to the
// upstream voice implementation until the Jetson becomes healthy again.
if ( window . SpeechRecognition || window . webkitSpeechRecognition ) {
modeBtn . removeEventListener ( 'click' , onVoiceClick , true ) ;
window . setTimeout ( function ( ) { modeBtn . click ( ) ; } , 50 ) ;
}
}
}
2026-08-23 18:29:02 -03:00
function websocketUrl ( path ) {
const protocol = window . location && window . location . protocol === 'https:' ? 'wss:' : 'ws:' ;
return protocol + '//' + window . location . host + path ;
}
function encodePcm16 ( samples ) {
const bytes = new ArrayBuffer ( samples . length * 2 ) ;
const view = new DataView ( bytes ) ;
for ( let index = 0 ; index < samples . length ; index += 1 ) {
const value = Math . max ( - 1 , Math . min ( 1 , samples [ index ] ) ) ;
view . setInt16 ( index * 2 , value < 0 ? value * 32768 : value * 32767 , true ) ;
}
return bytes ;
}
2026-08-23 22:13:52 -03:00
function pcm16WavBlob ( buffers , sampleRate ) {
const chunks = Array . isArray ( buffers ) ? buffers : [ ] ;
const pcmBytes = chunks . reduce ( function ( total , chunk ) { return total + ( chunk ? chunk . byteLength : 0 ) ; } , 0 ) ;
const wav = new ArrayBuffer ( 44 + pcmBytes ) ;
const view = new DataView ( wav ) ;
const write = function ( offset , value ) { for ( let index = 0 ; index < value . length ; index += 1 ) view . setUint8 ( offset + index , value . charCodeAt ( index ) ) ; } ;
write ( 0 , 'RIFF' ) ;
view . setUint32 ( 4 , 36 + pcmBytes , true ) ;
write ( 8 , 'WAVE' ) ;
write ( 12 , 'fmt ' ) ;
view . setUint32 ( 16 , 16 , true ) ;
view . setUint16 ( 20 , 1 , true ) ;
view . setUint16 ( 22 , 1 , true ) ;
view . setUint32 ( 24 , sampleRate , true ) ;
view . setUint32 ( 28 , sampleRate * 2 , true ) ;
view . setUint16 ( 32 , 2 , true ) ;
view . setUint16 ( 34 , 16 , true ) ;
write ( 36 , 'data' ) ;
view . setUint32 ( 40 , pcmBytes , true ) ;
const output = new Uint8Array ( wav , 44 ) ;
let offset = 0 ;
chunks . forEach ( function ( chunk ) { const bytes = new Uint8Array ( chunk ) ; output . set ( bytes , offset ) ; offset += bytes . length ; } ) ;
return new Blob ( [ wav ] , { type : 'audio/wav' } ) ;
}
window . _atlasPcm16WavBlob = pcm16WavBlob ;
2026-08-23 18:29:02 -03:00
function createResampler ( sourceRate , targetRate ) {
let tail = null ;
let position = 0 ;
const ratio = sourceRate / targetRate ;
return function ( samples ) {
const input = new Float32Array ( samples . length + ( tail === null ? 0 : 1 ) ) ;
let offset = 0 ;
if ( tail !== null ) { input [ 0 ] = tail ; offset = 1 ; }
input . set ( samples , offset ) ;
if ( input . length < 2 ) { tail = input . length ? input [ 0 ] : tail ; return new Float32Array ( 0 ) ; }
const output = [ ] ;
while ( position < input . length - 1 ) {
const left = Math . floor ( position ) ;
const fraction = position - left ;
output . push ( input [ left ] + ( ( input [ left + 1 ] - input [ left ] ) * fraction ) ) ;
position += ratio ;
}
position -= input . length - 1 ;
tail = input [ input . length - 1 ] ;
return new Float32Array ( output ) ;
} ;
}
function createStreamingSttSession ( turnId , context ) {
if ( ! streamingCapability . stt || ! window . WebSocket || ! window . location ) return null ;
let socket ;
let settled = false ;
let committed = false ;
let speculative = false ;
let workletReady = false ;
2026-08-23 22:13:52 -03:00
let partialRevision = - 1 ;
let lastPreflightText = '' ;
2026-08-23 18:29:02 -03:00
let queue = [ ] ;
2026-08-23 22:13:52 -03:00
let queuedBytes = 0 ;
let flushTimer = null ;
let archive = [ ] ;
let archiveBytes = 0 ;
let cancelled = false ;
2026-08-23 18:29:02 -03:00
let resolveFinal ;
let rejectFinal ;
const finalPromise = new Promise ( function ( resolve , reject ) { resolveFinal = resolve ; rejectFinal = reject ; } ) ;
// A transport can fail before recorder.onstop awaits it. Keep the rejection
// observed without changing what the later await receives.
finalPromise . catch ( function ( ) { } ) ;
const resample = createResampler ( context . sampleRate , 16000 ) ;
2026-08-23 22:13:52 -03:00
function clearFlushTimer ( ) {
if ( flushTimer ) { window . clearTimeout ( flushTimer ) ; flushTimer = null ; }
}
function clearQueue ( ) { queue = [ ] ; queuedBytes = 0 ; clearFlushTimer ( ) ; }
function clearArchive ( ) { archive = [ ] ; archiveBytes = 0 ; }
function archivePcm ( bytes ) {
if ( cancelled || ! bytes || ! bytes . byteLength || archiveBytes >= STT _MAX _ARCHIVE _BYTES ) return ;
const retained = Math . min ( bytes . byteLength , STT _MAX _ARCHIVE _BYTES - archiveBytes ) ;
const even = retained - ( retained % 2 ) ;
if ( ! even ) return ;
archive . push ( bytes . slice ( 0 , even ) ) ;
archiveBytes += even ;
}
2026-08-23 18:29:02 -03:00
function reject ( error ) {
if ( settled ) return ;
settled = true ;
2026-08-23 22:13:52 -03:00
cancelVoicePreflight ( turnId ) ;
clearQueue ( ) ;
2026-08-23 18:29:02 -03:00
rejectFinal ( error instanceof Error ? error : new Error ( String ( error || 'Streaming transcription failed' ) ) ) ;
}
function sendJson ( payload ) {
if ( socket && socket . readyState === 1 ) { socket . send ( JSON . stringify ( payload ) ) ; return true ; }
return false ;
}
2026-08-23 22:13:52 -03:00
function scheduleFlush ( ) {
if ( flushTimer || settled || ! queue . length || ! socket || socket . readyState !== 1 ) return ;
flushTimer = window . setTimeout ( function ( ) { flushTimer = null ; flush ( ) ; } , 20 ) ;
}
2026-08-23 18:29:02 -03:00
function flush ( ) {
2026-08-23 22:13:52 -03:00
clearFlushTimer ( ) ;
if ( settled ) { clearQueue ( ) ; return ; }
2026-08-23 18:29:02 -03:00
if ( ! socket || socket . readyState !== 1 ) return ;
2026-08-23 22:13:52 -03:00
while ( queue . length && socket . bufferedAmount < 524288 ) {
const bytes = queue . shift ( ) ;
queuedBytes = Math . max ( 0 , queuedBytes - bytes . byteLength ) ;
socket . send ( bytes ) ;
}
if ( queue . length ) scheduleFlush ( ) ;
2026-08-23 18:29:02 -03:00
}
try {
const csrf = String ( ( window . _ _HERMES _CONFIG _ _ && window . _ _HERMES _CONFIG _ _ . csrfToken ) || '' ) ;
const protocols = [ 'hermes-voice-v1' ] ;
if ( csrf ) protocols . push ( 'hermes-csrf.' + csrf ) ;
socket = new WebSocket ( websocketUrl ( STT _STREAM _PATH ) , protocols ) ;
socket . binaryType = 'arraybuffer' ;
} catch ( error ) {
reject ( error ) ;
return null ;
}
socket . onopen = function ( ) {
sendJson ( { type : 'start' , turn _id : turnId , format : 'pcm_s16le' , sample _rate : 16000 , language : 'auto' } ) ;
flush ( ) ;
} ;
socket . onmessage = function ( event ) {
if ( typeof event . data !== 'string' ) return ;
let payload ;
try { payload = JSON . parse ( event . data ) ; } catch ( _ ) { return ; }
if ( payload . turn _id !== turnId ) return ;
2026-08-23 22:13:52 -03:00
if ( payload . type === 'partial' && payload . rolling === true ) {
const revision = Number ( payload . revision ) ;
if ( Number . isFinite ( revision ) && revision > partialRevision ) {
partialRevision = revision ;
const stable = String ( payload . stable _transcript || '' ) . trim ( ) ;
2026-08-24 00:36:28 -03:00
const provisional = String ( payload . transcript || '' ) . trim ( ) ;
const visible = stable || provisional ;
if ( visible && active && captureTurnId === turnId && state === 'listening' ) {
const preview = visible . length > 72 ? visible . slice ( 0 , 69 ) + '…' : visible ;
label . textContent = 'Listening · ' + preview + ( stable ? '' : ' · provisional' ) ;
2026-08-23 22:13:52 -03:00
if ( stable !== lastPreflightText ) {
2026-08-24 00:36:28 -03:00
if ( stable ) {
lastPreflightText = stable ;
scheduleVoicePreflight ( turnId , revision , stable ) ;
}
2026-08-23 22:13:52 -03:00
}
}
}
} else if ( payload . type === 'final' && ! settled ) {
2026-08-23 18:29:02 -03:00
settled = true ;
2026-08-23 22:13:52 -03:00
cancelVoicePreflight ( turnId ) ;
clearQueue ( ) ;
clearArchive ( ) ;
2026-08-23 18:29:02 -03:00
resolveFinal ( { transcript : payload . transcript || '' , language : normalizeSttLanguage ( payload . language ) } ) ;
} else if ( payload . type === 'error' ) {
reject ( new Error ( payload . error || 'Streaming transcription failed' ) ) ;
}
} ;
socket . onerror = function ( ) { reject ( new Error ( 'Streaming transcription connection failed' ) ) ; } ;
socket . onclose = function ( ) { if ( ! settled ) reject ( new Error ( 'Streaming transcription closed before final result' ) ) ; } ;
return {
finalPromise : finalPromise ,
setWorkletReady : function ( value ) { workletReady = value ; } ,
push : function ( samples ) {
2026-08-23 22:13:52 -03:00
if ( cancelled || committed || ! workletReady ) return ;
2026-08-23 18:29:02 -03:00
const pcm = resample ( samples ) ;
if ( ! pcm . length ) return ;
2026-08-23 22:13:52 -03:00
const bytes = encodePcm16 ( pcm ) ;
archivePcm ( bytes ) ;
if ( settled ) return ;
if ( queuedBytes + bytes . byteLength > STT _MAX _QUEUED _BYTES ) {
reject ( new Error ( 'Streaming transcription backpressure limit exceeded' ) ) ;
if ( socket && socket . readyState < 2 ) socket . close ( 1008 , 'client backpressure' ) ;
return ;
}
queue . push ( bytes ) ;
queuedBytes += bytes . byteLength ;
2026-08-23 18:29:02 -03:00
flush ( ) ;
} ,
2026-08-23 22:13:52 -03:00
takeFallbackBlob : function ( ) {
if ( ! archiveBytes ) return null ;
const blob = pcm16WavBlob ( archive , 16000 ) ;
clearArchive ( ) ;
return blob ;
} ,
2026-08-23 18:29:02 -03:00
speculate : function ( ) {
if ( settled || committed || speculative ) return ;
speculative = true ;
sendJson ( { type : 'speculate' , turn _id : turnId } ) ;
} ,
resume : function ( ) {
if ( settled || committed || ! speculative ) return ;
speculative = false ;
sendJson ( { type : 'resume' , turn _id : turnId } ) ;
} ,
commit : function ( ) {
if ( settled || committed ) return finalPromise ;
committed = true ;
flush ( ) ;
const commitDeadline = Date . now ( ) + 30000 ;
const sendCommit = function ( ) {
if ( settled ) return ;
if ( ! socket || socket . readyState > 1 || Date . now ( ) >= commitDeadline ) {
reject ( new Error ( 'Streaming transcription closed before commit' ) ) ;
return ;
}
if ( socket . readyState !== 1 || queue . length ) { flush ( ) ; window . setTimeout ( sendCommit , 20 ) ; return ; }
if ( ! sendJson ( { type : 'commit' , turn _id : turnId } ) ) reject ( new Error ( 'Streaming transcription commit failed' ) ) ;
} ;
sendCommit ( ) ;
return finalPromise ;
} ,
cancel : function ( ) {
2026-08-23 22:13:52 -03:00
cancelled = true ;
cancelVoicePreflight ( turnId ) ;
clearQueue ( ) ;
clearArchive ( ) ;
2026-08-23 18:29:02 -03:00
if ( ! settled ) { sendJson ( { type : 'cancel' , turn _id : turnId } ) ; reject ( new Error ( 'Streaming transcription cancelled' ) ) ; }
if ( socket && socket . readyState < 2 ) socket . close ( 1000 , 'cancelled' ) ;
} ,
} ;
}
async function installCaptureWorklet ( context , source , session ) {
if ( ! session ) return false ;
if ( ! context . audioWorklet || ! window . AudioWorkletNode ) {
session . cancel ( ) ;
if ( streamingStt === session ) streamingStt = null ;
return false ;
}
try {
await context . audioWorklet . addModule ( WORKLET _URL ) ;
if ( ! streamingStt || streamingStt !== session ) return false ;
const node = new AudioWorkletNode ( context , 'atlas-pcm-capture' ) ;
const silentGain = context . createGain ( ) ;
silentGain . gain . value = 0 ;
let flushResolve = null ;
node . port . onmessage = function ( event ) {
if ( ! event . data ) return ;
if ( event . data . type === 'flushed' && flushResolve ) { flushResolve ( ) ; flushResolve = null ; return ; }
if ( ! streamingStt || streamingStt !== session || event . data . type !== 'pcm' ) return ;
session . push ( new Float32Array ( event . data . samples ) ) ;
} ;
node . _atlasFlush = function ( ) {
return new Promise ( function ( resolve ) { flushResolve = resolve ; node . port . postMessage ( { type : 'flush' } ) ; } ) ;
} ;
source . connect ( node ) ;
node . connect ( silentGain ) ;
silentGain . connect ( context . destination ) ;
captureNode = node ;
session . setWorkletReady ( true ) ;
return true ;
} catch ( _ ) {
session . cancel ( ) ;
if ( streamingStt === session ) streamingStt = null ;
return false ;
}
}
2026-08-23 22:13:52 -03:00
function cancelActiveModelTurn ( ) {
const sessionId = ( typeof S !== 'undefined' && S . session ) ? String ( S . session . session _id || '' ) : '' ;
const streamId = typeof S !== 'undefined' ? String ( S . activeStreamId || '' ) : '' ;
if ( ! streamId ) return { sessionId : sessionId , streamId : '' , controller : null , promise : Promise . resolve ( false ) } ;
const controller = createAbortController ( ) ;
const timer = window . setTimeout ( function ( ) { controller . abort ( ) ; } , 1800 ) ;
const promise = ( async function ( ) {
try {
const url = new URL ( 'api/chat/cancel?stream_id=' + encodeURIComponent ( streamId ) , document . baseURI || location . href ) . href ;
const response = await fetch ( url , { credentials : 'include' , signal : controller . signal } ) ;
let payload = null ;
try { payload = await response . json ( ) ; } catch ( _ ) { }
const currentSession = ( typeof S !== 'undefined' && S . session ) ? String ( S . session . session _id || '' ) : '' ;
if ( response . ok && payload && payload . cancelled === false && currentSession === sessionId && String ( S . activeStreamId || '' ) === streamId ) {
S . activeStreamId = null ;
if ( S . session ) S . session . active _stream _id = null ;
if ( typeof setBusy === 'function' ) setBusy ( false ) ; else S . busy = false ;
}
return ! ! response . ok ;
} catch ( _ ) {
return false ;
} finally {
window . clearTimeout ( timer ) ;
}
} ) ( ) ;
return { sessionId : sessionId , streamId : streamId , controller : controller , promise : promise } ;
}
function trimBargeLookback ( monitor ) {
const maximum = Math . max ( 1 , Math . round ( monitor . context . sampleRate * BARGE _LOOKBACK _MS / 1000 ) ) ;
while ( monitor . lookbackFrames > maximum && monitor . lookback . length ) {
const overflow = monitor . lookbackFrames - maximum ;
const first = monitor . lookback [ 0 ] ;
if ( first . length <= overflow ) { monitor . lookback . shift ( ) ; monitor . lookbackFrames -= first . length ; continue ; }
monitor . lookback [ 0 ] = first . slice ( overflow ) ;
monitor . lookbackFrames -= overflow ;
}
}
async function triggerBargeIn ( monitor ) {
if ( ! bargeMonitor || bargeMonitor !== monitor || monitor . cancelled || ! active || monitor . token !== generation ) return ;
bargeMonitor = null ;
if ( monitor . timer ) { window . clearInterval ( monitor . timer ) ; monitor . timer = null ; }
// Keep the old capture worklet alive until the new STT worklet is attached.
// Its lookback becomes an untrimmed handoff buffer so no syllables disappear
// during AudioWorklet/session setup.
monitor . handoff = true ;
const oldToken = monitor . token ;
generation += 1 ;
const token = generation ;
thinkingSession = null ;
thinkingTurnId = '' ;
suppressAutoRead = true ;
clearSttLanguage ( ) ;
stopResponseObserver ( ) ;
cancelThinkingCues ( ) ;
cancelSpeechTurn ( ) ;
stopPlayback ( ) ;
if ( typeof window . stopTTS === 'function' ) window . stopTTS ( ) ;
clearBargeCancellation ( ) ;
bargeCancelPromise = cancelActiveModelTurn ( ) ;
setState ( 'listening' , 'Listening — interrupted' ) ;
if ( oldToken === token ) return ;
startListening ( token , {
stream : monitor . stream ,
context : monitor . context ,
lookback : monitor . lookback ,
handoffMonitor : monitor ,
heardSpeech : true ,
requireStreamingLookback : true ,
} ) ;
}
async function startBargeMonitor ( token ) {
if ( ! active || token !== generation || ( state !== 'thinking' && state !== 'speaking' ) || bargeMonitor ) return ;
const monitor = { token : token , cancelled : false , handoff : false , aecUsable : true , timer : null , stream : null , context : null , source : null , node : null , silentGain : null , lookback : [ ] , lookbackFrames : 0 } ;
bargeMonitor = monitor ;
try {
monitor . stream = await acquireMicrophone ( ) ;
if ( ! active || token !== generation || bargeMonitor !== monitor ) { disposeBargeResources ( monitor , false ) ; return ; }
monitor . aecUsable = captureAecIsUsable ( monitor . stream ) ;
const Context = window . AudioContext || window . webkitAudioContext ;
try { monitor . context = new Context ( { latencyHint : 'interactive' } ) ; } catch ( _ ) { monitor . context = new Context ( ) ; }
const analyser = monitor . context . createAnalyser ( ) ;
analyser . fftSize = 1024 ;
monitor . source = monitor . context . createMediaStreamSource ( monitor . stream ) ;
monitor . source . connect ( analyser ) ;
if ( monitor . context . audioWorklet && window . AudioWorkletNode ) {
await monitor . context . audioWorklet . addModule ( WORKLET _URL ) ;
if ( bargeMonitor !== monitor || monitor . cancelled ) { disposeBargeResources ( monitor , false ) ; return ; }
monitor . node = new AudioWorkletNode ( monitor . context , 'atlas-pcm-capture' ) ;
monitor . silentGain = monitor . context . createGain ( ) ;
monitor . silentGain . gain . value = 0 ;
monitor . node . port . onmessage = function ( event ) {
if ( monitor . cancelled || ( ! monitor . handoff && bargeMonitor !== monitor ) || ! event . data || event . data . type !== 'pcm' ) return ;
const samples = new Float32Array ( event . data . samples ) ;
monitor . lookback . push ( samples ) ;
monitor . lookbackFrames += samples . length ;
if ( ! monitor . handoff ) trimBargeLookback ( monitor ) ;
} ;
monitor . source . connect ( monitor . node ) ;
monitor . node . connect ( monitor . silentGain ) ;
monitor . silentGain . connect ( monitor . context . destination ) ;
}
await monitor . context . resume ( ) ;
const samples = new Uint8Array ( analyser . fftSize ) ;
let noiseFloor = 0.008 ;
let voiceFrames = 0 ;
let ducked = false ;
let speechArmAt = Date . now ( ) + 100 ;
let playbackWasActive = false ;
monitor . timer = window . setInterval ( function ( ) {
if ( ! active || token !== generation || bargeMonitor !== monitor || ( state !== 'thinking' && state !== 'speaking' ) ) { stopBargeMonitor ( ) ; return ; }
analyser . getByteTimeDomainData ( samples ) ;
let energy = 0 ;
for ( let index = 0 ; index < samples . length ; index += 1 ) { const value = ( samples [ index ] - 128 ) / 128 ; energy += value * value ; }
const rms = Math . sqrt ( energy / samples . length ) ;
const now = Date . now ( ) ;
const playbackActive = indicator . classList . contains ( 'is-playing' ) || ! ! currentAudio || ! ! ( playbackSession && playbackSession . node ) || ! ! ( thinkingCue && thinkingCue . node ) ;
if ( playbackActive && ! playbackWasActive ) {
// AEC needs a brief convergence window when local speech starts. PCM
// lookback preserves a real interruption spoken during this guard.
// A brief cached thinking cue uses the shorter guard so an "um" can
// still interrupt it; answer playback gets the conservative window.
speechArmAt = now + ( state === 'thinking' ? 150 : 450 ) ;
voiceFrames = 0 ;
if ( ducked ) { ducked = false ; setPlaybackDucked ( false ) ; }
} else if ( ! playbackActive && playbackWasActive ) {
speechArmAt = now + 100 ;
}
playbackWasActive = playbackActive ;
const threshold = Math . max ( 0.05 , ( noiseFloor * 3 ) + 0.008 ) ;
// Never treat speaker leakage as an interruption when the browser says
// the requested AEC was not applied. Quiet Thinking remains fully
// interruptible, and unknown/unsupported settings preserve normal use.
const automaticBargeAllowed = ! playbackActive || monitor . aecUsable ;
const voiceNow = automaticBargeAllowed && now >= speechArmAt && rms > threshold ;
if ( ! voiceNow ) noiseFloor = ( noiseFloor * 0.975 ) + ( Math . min ( rms , threshold ) * 0.025 ) ;
voiceFrames = voiceNow ? Math . min ( voiceFrames + 1 , BARGE _TRIGGER _FRAMES ) : Math . max ( voiceFrames - 1 , 0 ) ;
if ( voiceFrames >= BARGE _DUCK _FRAMES && ! ducked ) { ducked = true ; setPlaybackDucked ( true ) ; }
if ( ! voiceFrames && ducked ) { ducked = false ; setPlaybackDucked ( false ) ; }
if ( voiceFrames >= BARGE _TRIGGER _FRAMES ) triggerBargeIn ( monitor ) ;
} , 50 ) ;
} catch ( _ ) {
if ( bargeMonitor === monitor ) bargeMonitor = null ;
disposeBargeResources ( monitor , false ) ;
// Barge-in is an optional full-duplex enhancement. The normal finalized
// microphone turn remains available if the browser rejects concurrent
// capture or AudioWorklet initialization.
}
}
async function transcribeStreamingOrFallback ( blob , token , session , allowContainerFallback ) {
2026-08-23 18:29:02 -03:00
if ( ! active || token !== generation ) return ;
setState ( 'transcribing' ) ;
2026-08-23 22:13:52 -03:00
let pcmFallback = null ;
2026-08-23 18:29:02 -03:00
if ( session ) {
try {
const result = await session . commit ( ) ;
if ( ! String ( result . transcript || '' ) . trim ( ) ) throw new Error ( 'Streaming transcription returned no final text' ) ;
if ( active && token === generation ) {
streamingStt = null ;
sendTranscript ( result . transcript , token , result . language ) ;
return ;
}
} catch ( _ ) {
// The finalized browser container below is the quality-preserving path
// whenever rolling PCM transport or speculative inference fails.
2026-08-23 22:13:52 -03:00
if ( typeof session . takeFallbackBlob === 'function' ) pcmFallback = session . takeFallbackBlob ( ) ;
2026-08-23 18:29:02 -03:00
}
if ( streamingStt === session ) streamingStt = null ;
}
2026-08-23 22:13:52 -03:00
if ( pcmFallback ) {
transcribe ( pcmFallback , token ) ;
return ;
}
if ( allowContainerFallback === false ) {
toast ( 'I missed the start of that interruption. Please repeat it.' ) ;
restartSoon ( token , 250 ) ;
return ;
}
2026-08-23 18:29:02 -03:00
transcribe ( blob , token ) ;
}
2026-08-23 22:13:52 -03:00
async function startListening ( token , reusedCapture ) {
2026-08-10 00:42:37 -03:00
if ( ! active || token !== generation ) return ;
stopCapture ( ) ;
2026-08-23 18:29:02 -03:00
stopPlayback ( ) ;
cancelSpeechTurn ( ) ;
2026-08-21 11:52:49 +00:00
clearSttLanguage ( ) ;
2026-08-23 22:13:52 -03:00
captureTurnId = ( voiceTabNonce ? voiceTabNonce + '-' : '' ) + String ( token ) + '-' + String ( ++ turnSequence ) ;
2026-08-10 00:42:37 -03:00
setState ( 'listening' ) ;
try {
2026-08-23 22:13:52 -03:00
const capture = reusedCapture && reusedCapture . stream ? reusedCapture . stream : await acquireMicrophone ( ) ;
if ( ! active || token !== generation ) {
if ( reusedCapture && reusedCapture . handoffMonitor ) disposeBargeResources ( reusedCapture . handoffMonitor , false ) ;
else capture . getTracks ( ) . forEach ( function ( track ) { track . stop ( ) ; } ) ;
return ;
}
2026-08-10 00:42:37 -03:00
stream = capture ;
const Context = window . AudioContext || window . webkitAudioContext ;
2026-08-23 22:13:52 -03:00
if ( reusedCapture && reusedCapture . context ) {
audioContext = reusedCapture . context ;
} else {
try {
// Let the browser's native resampler produce Whisper's 16 kHz input.
audioContext = new Context ( { sampleRate : 16000 , latencyHint : 'interactive' } ) ;
} catch ( _ ) {
audioContext = new Context ( ) ;
}
2026-08-23 18:29:02 -03:00
}
2026-08-10 00:42:37 -03:00
const analyser = audioContext . createAnalyser ( ) ;
2026-08-10 01:47:51 -03:00
analyser . fftSize = 2048 ;
const highpass = audioContext . createBiquadFilter ( ) ;
highpass . type = 'highpass' ;
highpass . frequency . value = 140 ;
highpass . Q . value = 0.7 ;
2026-08-23 18:29:02 -03:00
const mediaSource = audioContext . createMediaStreamSource ( stream ) ;
mediaSource . connect ( highpass ) ;
2026-08-10 01:47:51 -03:00
highpass . connect ( analyser ) ;
2026-08-23 18:29:02 -03:00
streamingStt = createStreamingSttSession ( captureTurnId , audioContext ) ;
// Whisper receives the browser's full-band processed microphone signal;
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
// needlessly altered before final recognition.
if ( streamingStt ) await installCaptureWorklet ( audioContext , mediaSource , streamingStt ) ;
2026-08-23 22:13:52 -03:00
if ( streamingStt && reusedCapture && Array . isArray ( reusedCapture . lookback ) ) {
reusedCapture . lookback . forEach ( function ( samples ) { streamingStt . push ( samples ) ; } ) ;
}
if ( reusedCapture && reusedCapture . handoffMonitor ) {
disposeBargeResources ( reusedCapture . handoffMonitor , true ) ;
reusedCapture . handoffMonitor = null ;
}
if ( reusedCapture && reusedCapture . requireStreamingLookback && ! streamingStt ) {
toast ( 'Streaming transcription is unavailable. Please repeat your interruption.' ) ;
stopCapture ( ) ;
restartSoon ( token , 250 ) ;
return ;
}
2026-08-10 00:42:37 -03:00
const samples = new Uint8Array ( analyser . fftSize ) ;
2026-08-21 11:12:15 +00:00
const mimeTypes = [ 'audio/webm;codecs=opus' , 'audio/ogg;codecs=opus' , 'audio/mp4;codecs=mp4a.40.2' , 'audio/mp4' , 'audio/webm' ] ;
2026-08-10 00:42:37 -03:00
const mime = mimeTypes . find ( function ( value ) { return MediaRecorder . isTypeSupported ( value ) ; } ) || '' ;
const chunks = [ ] ;
2026-08-23 22:13:52 -03:00
let heardSpeech = ! ! ( reusedCapture && reusedCapture . heardSpeech ) ;
let voiceFrames = heardSpeech ? 3 : 0 ;
2026-08-10 01:47:51 -03:00
let noiseFloor = 0.008 ;
2026-08-10 00:42:37 -03:00
let lastSpeech = Date . now ( ) ;
2026-08-23 18:29:02 -03:00
let speculative = false ;
2026-08-10 00:42:37 -03:00
const started = Date . now ( ) ;
recorder = new MediaRecorder ( stream , mime ? { mimeType : mime } : undefined ) ;
2026-08-21 11:12:15 +00:00
let recordedMime = recorder . mimeType || mime || '' ;
2026-08-10 01:47:51 -03:00
recorder . ondataavailable = function ( event ) {
if ( ! event . data || ! event . data . size ) return ;
2026-08-21 11:12:15 +00:00
if ( event . data . type ) recordedMime = event . data . type ;
2026-08-23 15:59:43 -03:00
chunks . push ( event . data ) ;
2026-08-10 01:47:51 -03:00
} ;
2026-08-23 18:29:02 -03:00
recorder . onstop = async function ( ) {
2026-08-10 00:42:37 -03:00
if ( vadTimer ) { clearInterval ( vadTimer ) ; vadTimer = null ; }
2026-08-23 18:29:02 -03:00
const finalCaptureNode = captureNode ;
if ( finalCaptureNode && typeof finalCaptureNode . _atlasFlush === 'function' ) {
await Promise . race ( [
finalCaptureNode . _atlasFlush ( ) ,
new Promise ( function ( resolve ) { window . setTimeout ( resolve , 100 ) ; } ) ,
] ) ;
}
2026-08-10 00:42:37 -03:00
const recordedStream = stream ;
stream = null ;
if ( recordedStream ) recordedStream . getTracks ( ) . forEach ( function ( track ) { track . stop ( ) ; } ) ;
2026-08-23 18:29:02 -03:00
const context = audioContext ;
audioContext = null ;
if ( context ) { try { context . close ( ) ; } catch ( _ ) { } }
captureNode = null ;
2026-08-10 00:42:37 -03:00
recorder = null ;
2026-08-23 18:29:02 -03:00
if ( ! active || token !== generation ) { cancelStreamingStt ( ) ; return ; }
if ( ! heardSpeech || ! chunks . length ) { cancelStreamingStt ( ) ; restartSoon ( token , 300 ) ; return ; }
const session = streamingStt ;
2026-08-23 22:13:52 -03:00
transcribeStreamingOrFallback (
new Blob ( chunks , { type : recordedMime || 'audio/webm' } ) ,
token ,
session ,
! ( reusedCapture && reusedCapture . requireStreamingLookback )
) ;
2026-08-10 00:42:37 -03:00
} ;
2026-08-23 15:59:43 -03:00
// Ask the browser for one finalized container at stop. Android Chromium
// can emit timeslice fragments without a reusable EBML initialization
// header; concatenating those fragments made otherwise valid recordings
// intermittently unreadable by ffmpeg. A bounded 90-second Opus capture
// is small enough to retain as one browser-owned recording.
recorder . start ( ) ;
2026-08-23 16:50:48 -03:00
const silenceMs = Math . max ( 900 , parseInt ( localStorage . getItem ( 'hermes-voice-silence-ms' ) || '1100' , 10 ) || 1100 ) ;
2026-08-23 18:29:02 -03:00
const speculateMs = Math . min ( silenceMs - 250 , Math . max ( 450 , Math . round ( silenceMs * 0.55 ) ) ) ;
2026-08-10 00:42:37 -03:00
vadTimer = window . setInterval ( function ( ) {
if ( ! active || token !== generation || ! recorder || recorder . state === 'inactive' ) return ;
analyser . getByteTimeDomainData ( samples ) ;
let energy = 0 ;
2026-08-23 18:29:02 -03:00
for ( let index = 0 ; index < samples . length ; index += 1 ) {
const normalized = ( samples [ index ] - 128 ) / 128 ;
2026-08-10 00:42:37 -03:00
energy += normalized * normalized ;
}
const rms = Math . sqrt ( energy / samples . length ) ;
2026-08-20 23:13:51 +00:00
updateInputLevel ( rms ) ;
2026-08-10 00:42:37 -03:00
const now = Date . now ( ) ;
2026-08-10 01:47:51 -03:00
const speechThreshold = Math . max ( 0.04 , noiseFloor * 2.4 + 0.006 ) ;
const voiceNow = rms > speechThreshold ;
if ( ! heardSpeech && ! voiceNow ) { noiseFloor = ( noiseFloor * 0.94 ) + ( rms * 0.06 ) ; }
voiceFrames = voiceNow ? Math . min ( voiceFrames + 1 , 5 ) : Math . max ( voiceFrames - 1 , 0 ) ;
if ( ! heardSpeech && voiceFrames >= 3 ) {
heardSpeech = true ;
lastSpeech = now ;
} else if ( heardSpeech && voiceNow ) {
lastSpeech = now ;
2026-08-23 18:29:02 -03:00
if ( speculative && streamingStt ) { streamingStt . resume ( ) ; speculative = false ; }
}
if ( heardSpeech && ! voiceNow && ! speculative && ( now - lastSpeech ) >= speculateMs && streamingStt ) {
streamingStt . speculate ( ) ;
speculative = true ;
2026-08-10 01:47:51 -03:00
}
2026-08-10 00:42:37 -03:00
const finished = heardSpeech && ( now - lastSpeech ) >= silenceMs ;
const timedOut = now - started >= 90000 ;
const idle = ( ! heardSpeech ) && ( now - started ) >= 20000 ;
if ( finished || timedOut || idle ) {
if ( vadTimer ) { clearInterval ( vadTimer ) ; vadTimer = null ; }
try { recorder . stop ( ) ; } catch ( _ ) { }
}
} , 100 ) ;
} catch ( error ) {
2026-08-23 22:13:52 -03:00
if ( reusedCapture && reusedCapture . handoffMonitor ) {
disposeBargeResources ( reusedCapture . handoffMonitor , true ) ;
reusedCapture . handoffMonitor = null ;
}
2026-08-10 00:42:37 -03:00
if ( ! active || token !== generation ) return ;
2026-08-20 23:13:51 +00:00
const message = errorMessage ( error , 'Microphone permission is required' ) ;
showUnavailable ( message ) ;
toast ( message ) ;
2026-08-10 00:42:37 -03:00
}
}
2026-08-23 22:13:52 -03:00
function stripHttpUrlsForSpeech ( text ) {
return String ( text || '' ) . replace ( /(^|\s+[([{]|\s+|[([{"'])https?:\/\/[^\s<>"']+/gi , function ( match , prefix ) {
let address = match . slice ( prefix . length ) ;
let suffix = '' ;
while ( address ) {
const last = address . slice ( - 1 ) ;
let trailing = /[.,!?;:…,。!?;:]/ . test ( last ) ;
if ( last === ')' ) trailing = ( address . match ( /\)/g ) || [ ] ) . length > ( address . match ( /\(/g ) || [ ] ) . length ;
if ( last === ']' ) trailing = ( address . match ( /\]/g ) || [ ] ) . length > ( address . match ( /\[/g ) || [ ] ) . length ;
if ( last === '}' ) trailing = ( address . match ( /\}/g ) || [ ] ) . length > ( address . match ( /\{/g ) || [ ] ) . length ;
if ( ! trailing ) break ;
suffix = last + suffix ;
address = address . slice ( 0 , - 1 ) ;
}
const pairs = { '(' : ')' , '[' : ']' , '{' : '}' } ;
const opening = prefix . slice ( - 1 ) ;
if ( pairs [ opening ] && suffix . startsWith ( pairs [ opening ] ) ) {
prefix = prefix . slice ( 0 , - 1 ) ;
suffix = suffix . slice ( 1 ) ;
}
if ( /^\s+$/ . test ( prefix ) && /^[.,!?;:…,。!?;:]/ . test ( suffix ) ) prefix = '' ;
return prefix + suffix ;
} ) ;
}
window . _atlasStripHttpUrlsForSpeech = stripHttpUrlsForSpeech ;
2026-08-10 00:42:37 -03:00
function cleanForSpeech ( text ) {
2026-08-23 22:13:52 -03:00
const cleaned = typeof window . _stripForTTS === 'function' ? window . _stripForTTS ( text ) : String ( text || '' ) . replace ( /```[\s\S]*?```/g , ' code block ' ) ;
return stripHttpUrlsForSpeech ( cleaned ) . replace ( /\s+/g , ' ' ) . trim ( ) ;
2026-08-10 00:42:37 -03:00
}
2026-08-23 18:29:02 -03:00
function sentenceEnd ( text ) {
const source = String ( text || '' ) ;
const boundary = /[.!?…]+["')\]}]*(?:\s|$)/g ;
const abbreviation = /(?:\b(?:mr|mrs|ms|dr|prof|sr|jr|st|vs|etc|e\.g|i\.e)|\b[A-Z])\.$/i ;
let match ;
while ( ( match = boundary . exec ( source ) ) ) {
const end = match . index + match [ 0 ] . trimEnd ( ) . length ;
const prefix = source . slice ( 0 , end ) . replace ( /["')\]}]+$/ , '' ) ;
if ( prefix . endsWith ( '.' ) && abbreviation . test ( prefix ) ) continue ;
return end ;
}
return - 1 ;
}
function preferredCut ( text , min , target , max ) {
const bounded = text . slice ( 0 , max ) ;
const candidates = [ ] ;
const punctuation = /[,;:—–.!?…]+["')\]}]*(?:\s|$)/g ;
let match ;
while ( ( match = punctuation . exec ( bounded ) ) ) {
const end = match . index + match [ 0 ] . trimEnd ( ) . length ;
if ( end >= min ) candidates . push ( { end : end , weight : Math . abs ( end - target ) } ) ;
}
if ( candidates . length ) {
candidates . sort ( function ( left , right ) { return left . weight - right . weight ; } ) ;
return candidates [ 0 ] . end ;
}
const spaces = [ ] ;
const whitespace = /\s+/g ;
while ( ( match = whitespace . exec ( bounded ) ) ) { if ( match . index >= min ) spaces . push ( match . index ) ; }
if ( spaces . length ) {
spaces . sort ( function ( left , right ) { return Math . abs ( left - target ) - Math . abs ( right - target ) ; } ) ;
return spaces [ 0 ] ;
}
return Math . min ( max , text . length ) ;
}
function adaptiveChunks ( text , final , firstChunk ) {
const source = String ( text || '' ) . trim ( ) ;
if ( ! source ) return { chunks : [ ] , consumed : 0 } ;
const chunks = [ ] ;
let offset = 0 ;
if ( firstChunk !== false ) {
const firstSentence = sentenceEnd ( source ) ;
if ( firstSentence < 0 && ! final ) return { chunks : [ ] , consumed : 0 } ;
const firstAvailable = firstSentence >= 0 ? firstSentence : source . length ;
const firstSlice = source . slice ( 0 , firstAvailable ) ;
const firstCut = firstSlice . length <= 60 ? firstSlice . length : preferredCut ( firstSlice , 40 , 52 , 60 ) ;
chunks . push ( source . slice ( 0 , firstCut ) . trim ( ) ) ;
offset = firstCut ;
}
while ( offset < source . length ) {
const remaining = source . slice ( offset ) . trimStart ( ) ;
offset = source . length - remaining . length ;
if ( ! remaining ) break ;
if ( remaining . length < 100 ) {
const complete = sentenceEnd ( remaining ) ;
if ( ! final && complete < 0 ) break ;
const end = final ? remaining . length : complete ;
chunks . push ( remaining . slice ( 0 , end ) . trim ( ) ) ;
offset += end ;
continue ;
}
const cut = preferredCut ( remaining , 100 , 120 , 140 ) ;
if ( ! final && cut === remaining . length && remaining . length < 140 ) break ;
chunks . push ( remaining . slice ( 0 , cut ) . trim ( ) ) ;
offset += cut ;
}
return { chunks : chunks . filter ( Boolean ) , consumed : offset } ;
}
// Export the deterministic splitter as a narrow diagnostic/test seam.
window . _atlasAdaptiveChunks = function ( text , final ) { return adaptiveChunks ( text , final !== false , true ) . chunks ; } ;
function playBlob ( blob , token ) {
2026-08-10 00:42:37 -03:00
return new Promise ( function ( resolve , reject ) {
if ( ! active || token !== generation ) { resolve ( ) ; return ; }
2026-08-23 18:29:02 -03:00
const session = playbackSession ;
2026-08-10 00:42:37 -03:00
const url = URL . createObjectURL ( blob ) ;
const audio = new Audio ( url ) ;
currentAudio = audio ;
2026-08-23 18:29:02 -03:00
let settled = false ;
function cleanup ( callback , value ) {
if ( settled ) return ;
settled = true ;
2026-08-10 00:42:37 -03:00
if ( currentAudio === audio ) currentAudio = null ;
2026-08-23 18:29:02 -03:00
if ( session && session . blobWake === cancel ) session . blobWake = null ;
2026-08-20 23:13:51 +00:00
indicator . classList . remove ( 'is-playing' ) ;
2026-08-10 00:42:37 -03:00
URL . revokeObjectURL ( url ) ;
2026-08-23 18:29:02 -03:00
callback ( value ) ;
2026-08-10 00:42:37 -03:00
}
2026-08-23 18:29:02 -03:00
function cancel ( ) { cleanup ( resolve ) ; }
if ( session ) session . blobWake = cancel ;
audio . onended = function ( ) { cleanup ( resolve ) ; } ;
audio . onerror = function ( ) { cleanup ( reject , new Error ( 'Local speech playback failed' ) ) ; } ;
2026-08-20 23:13:51 +00:00
audio . play ( ) . then ( function ( ) {
if ( active && token === generation && currentAudio === audio ) indicator . classList . add ( 'is-playing' ) ;
2026-08-23 18:29:02 -03:00
} ) . catch ( function ( error ) { cleanup ( reject , error ) ; } ) ;
2026-08-10 00:42:37 -03:00
} ) ;
}
2026-08-23 18:29:02 -03:00
function ttsRequest ( chunk , language , turnId ) {
const request = { text : chunk , engine : 'atlas' , turn _id : turnId } ;
if ( language ) request . language = language ;
return request ;
}
2026-08-23 22:13:52 -03:00
async function fetchSpeech ( chunk , language , turnId , token ) {
2026-08-21 11:52:49 +00:00
// `language` is only ever the private STT result for this turn. When it is
// absent the field is omitted entirely and the server picks English.
2026-08-23 22:13:52 -03:00
if ( ! active || token !== generation ) throw cancelledError ( 'Speech turn cancelled' ) ;
const session = playbackSession ;
if ( ! session || session . cancelled || session . turnId !== turnId ) throw cancelledError ( 'Speech turn cancelled' ) ;
const controller = createAbortController ( ) ;
session . controllers . add ( controller ) ;
try {
const response = await fetch ( '/api/tts' , {
method : 'POST' ,
headers : { 'Content-Type' : 'application/json' } ,
body : JSON . stringify ( ttsRequest ( chunk , language , turnId ) ) ,
signal : controller . signal ,
} ) ;
if ( ! response . ok ) {
const payload = await response . json ( ) . catch ( function ( ) { return { } ; } ) ;
throw new Error ( payload . error || ( 'Local speech request failed: ' + response . status ) ) ;
}
const blob = await response . blob ( ) ;
if ( ! active || token !== generation || session . cancelled || playbackSession !== session ) throw cancelledError ( 'Speech turn cancelled' ) ;
return blob ;
} finally {
session . controllers . delete ( controller ) ;
2026-08-10 01:21:51 -03:00
}
2026-08-23 22:13:52 -03:00
}
function cuePoolOffset ( turnId , length ) {
let hash = 0 ;
const source = String ( turnId || '' ) ;
for ( let index = 0 ; index < source . length ; index += 1 ) hash = ( ( hash * 31 ) + source . charCodeAt ( index ) ) >>> 0 ;
return length ? hash % length : 0 ;
}
function thinkingCueStillOwned ( cue ) {
return ! ! ( cue && ! cue . cancelled && thinkingCue === cue && active && cue . token === generation && state === 'thinking' ) ;
}
function scheduleNextThinkingCue ( cue , delay ) {
if ( ! thinkingCueStillOwned ( cue ) || cue . issued >= cue . pool . length ) return ;
cue . timer = window . setTimeout ( function ( ) {
cue . timer = null ;
issueThinkingCue ( cue ) ;
} , delay ) ;
}
async function issueThinkingCue ( cue ) {
if ( ! thinkingCueStillOwned ( cue ) || cue . issued >= cue . pool . length ) return ;
const entry = cue . pool [ ( cue . offset + cue . issued ) % cue . pool . length ] ;
const cueNumber = cue . issued + 1 ;
cue . issued += 1 ;
const controller = createAbortController ( ) ;
cue . controller = controller ;
try {
const request = ttsRequest ( entry . text , cue . language , cue . turnId + ':thinking-cue:' + cueNumber ) ;
request . cue _id = entry . id ;
const response = await fetch ( TTS _STREAM _URL , {
method : 'POST' ,
headers : { 'Content-Type' : 'application/json' , 'Accept' : 'audio/pcm' } ,
body : JSON . stringify ( request ) ,
signal : controller . signal ,
} ) ;
if ( ! response . ok || ! response . body ) throw new Error ( 'Cached thinking cue unavailable' ) ;
const sampleRate = parseInt ( response . headers . get ( 'X-Audio-Sample-Rate' ) || '22050' , 10 ) ;
if ( ! Number . isFinite ( sampleRate ) || sampleRate < 8000 || sampleRate > 96000 ) throw new Error ( 'Cached thinking cue metadata invalid' ) ;
if ( ! thinkingCueStillOwned ( cue ) ) return ;
const Context = window . AudioContext || window . webkitAudioContext ;
try { cue . context = new Context ( { sampleRate : sampleRate , latencyHint : 'interactive' } ) ; } catch ( _ ) { cue . context = new Context ( ) ; }
await cue . context . audioWorklet . addModule ( WORKLET _URL ) ;
if ( ! thinkingCueStillOwned ( cue ) ) return ;
cue . node = new AudioWorkletNode ( cue . context , 'atlas-pcm-playback' ) ;
cue . gain = cue . context . createGain ( ) ;
cue . node . connect ( cue . gain ) ;
cue . gain . connect ( cue . context . destination ) ;
const drained = new Promise ( function ( resolve ) {
cue . audioWake = resolve ;
cue . node . port . onmessage = function ( event ) {
if ( ( event . data || { } ) . type === 'drained' ) {
const wake = cue . audioWake ;
cue . audioWake = null ;
if ( wake ) wake ( ) ;
}
} ;
} ) ;
await cue . context . resume ( ) ;
indicator . classList . add ( 'is-playing' ) ;
const reader = response . body . getReader ( ) ;
let carry = null ;
while ( thinkingCueStillOwned ( cue ) ) {
const result = await reader . read ( ) ;
if ( result . done ) break ;
let bytes = result . value ;
if ( carry !== null ) { const joined = new Uint8Array ( bytes . length + 1 ) ; joined [ 0 ] = carry ; joined . set ( bytes , 1 ) ; bytes = joined ; carry = null ; }
if ( bytes . length % 2 ) { carry = bytes [ bytes . length - 1 ] ; bytes = bytes . slice ( 0 , - 1 ) ; }
if ( ! bytes . length ) continue ;
const copy = bytes . buffer . slice ( bytes . byteOffset , bytes . byteOffset + bytes . byteLength ) ;
cue . node . port . postMessage ( { type : 'push' , samples : copy , sampleRate : sampleRate } , [ copy ] ) ;
}
if ( thinkingCueStillOwned ( cue ) ) {
cue . node . port . postMessage ( { type : 'end' } ) ;
await drained ;
} else {
try { await reader . cancel ( ) ; } catch ( _ ) { }
}
cue . controller = null ;
if ( cue . node ) { try { cue . node . disconnect ( ) ; } catch ( _ ) { } cue . node = null ; }
if ( cue . context ) { try { cue . context . close ( ) ; } catch ( _ ) { } cue . context = null ; }
cue . gain = null ;
indicator . classList . remove ( 'is-playing' ) ;
if ( thinkingCueStillOwned ( cue ) ) scheduleNextThinkingCue ( cue , THINKING _CUE _INTERVAL _MS ) ;
} catch ( error ) {
cue . controller = null ;
if ( cue . cancelled || ( error && error . name === 'AbortError' ) ) return ;
// Cues are optional. A playback or synthesis failure must not disturb the
// answer, retry noisily, or disable the primary hands-free conversation.
if ( thinkingCue === cue ) cancelThinkingCues ( ) ;
}
}
function scheduleThinkingCues ( token , language , turnId ) {
cancelThinkingCues ( ) ;
if ( ! active || token !== generation || state !== 'thinking' ) return ;
// Missing/unknown is the same fail-safe route as answer TTS: English/Amy.
// Known STT languages remain bound exactly to their turn voice.
const localized = normalizeSttLanguage ( language ) || 'en' ;
const pool = THINKING _CUE _POOLS [ localized ] ;
const cue = {
token : token ,
turnId : String ( turnId || token ) ,
language : localized ,
pool : pool ,
offset : cuePoolOffset ( turnId , pool . length ) ,
issued : 0 ,
timer : null ,
controller : null ,
audioWake : null ,
context : null ,
node : null ,
gain : null ,
cancelled : false ,
} ;
thinkingCue = cue ;
scheduleNextThinkingCue ( cue , THINKING _CUE _FIRST _MS ) ;
2026-08-10 01:21:51 -03:00
}
2026-08-23 18:29:02 -03:00
async function prepareSpeech ( chunk , language , turnId , token ) {
if ( streamingCapability . tts && window . ReadableStream && window . AudioWorkletNode ) {
2026-08-23 22:13:52 -03:00
const controller = createAbortController ( ) ;
2026-08-23 18:29:02 -03:00
if ( playbackSession ) playbackSession . controllers . add ( controller ) ;
try {
const response = await fetch ( TTS _STREAM _URL , {
method : 'POST' ,
headers : { 'Content-Type' : 'application/json' , 'Accept' : 'audio/pcm' } ,
body : JSON . stringify ( ttsRequest ( chunk , language , turnId ) ) ,
signal : controller . signal ,
} ) ;
if ( ! response . ok || ! response . body ) throw new Error ( 'Streaming speech unavailable' ) ;
const sampleRate = parseInt ( response . headers . get ( 'X-Audio-Sample-Rate' ) || String ( streamingCapability . tts . sample _rate || 22050 ) , 10 ) ;
const channels = parseInt ( response . headers . get ( 'X-Audio-Channels' ) || '1' , 10 ) ;
if ( ! Number . isFinite ( sampleRate ) || sampleRate < 8000 || sampleRate > 96000 || channels !== 1 ) throw new Error ( 'Unsupported streaming speech format' ) ;
return { kind : 'pcm' , response : response , sampleRate : sampleRate , controller : controller , token : token , chunk : chunk , language : language , turnId : turnId , started : false } ;
} catch ( error ) {
if ( playbackSession ) playbackSession . controllers . delete ( controller ) ;
if ( error && error . name === 'AbortError' ) throw error ;
// Disable only this optional transport for the browser session. The
// complete WAV endpoint preserves voice quality and availability.
streamingCapability . tts = null ;
}
}
2026-08-23 22:13:52 -03:00
return { kind : 'blob' , blob : await fetchSpeech ( chunk , language , turnId , token ) } ;
2026-08-23 18:29:02 -03:00
}
2026-08-23 22:13:52 -03:00
async function ensurePcmPlayback ( asset , token ) {
if ( ! active || token !== generation ) return null ;
const session = playbackSession ;
if ( ! session || session . cancelled ) return null ;
if ( session . node ) {
if ( session . sampleRate !== asset . sampleRate ) throw new Error ( 'Streaming speech sample rate changed mid-turn' ) ;
return session ;
}
2026-08-23 18:29:02 -03:00
const Context = window . AudioContext || window . webkitAudioContext ;
let context ;
try {
// Keep Piper at its native rate and let the browser/audio device own any
// final hardware conversion; the worklet interpolator is a fallback.
context = new Context ( { sampleRate : asset . sampleRate , latencyHint : 'interactive' } ) ;
} catch ( _ ) {
context = new Context ( ) ;
2026-08-10 00:42:37 -03:00
}
2026-08-23 18:29:02 -03:00
await context . audioWorklet . addModule ( WORKLET _URL ) ;
if ( ! active || token !== generation ) { context . close ( ) ; return ; }
const node = new AudioWorkletNode ( context , 'atlas-pcm-playback' ) ;
if ( ! session || session . cancelled ) { context . close ( ) ; return ; }
2026-08-23 22:13:52 -03:00
const gain = context . createGain ( ) ;
2026-08-23 18:29:02 -03:00
session . context = context ;
session . node = node ;
2026-08-23 22:13:52 -03:00
session . gain = gain ;
session . sampleRate = asset . sampleRate ;
session . bufferedFrames = 0 ;
session . playbackEnded = false ;
node . connect ( gain ) ;
gain . connect ( context . destination ) ;
2026-08-23 18:29:02 -03:00
await context . resume ( ) ;
2026-08-23 22:13:52 -03:00
session . drained = new Promise ( function ( resolve , reject ) {
2026-08-23 18:29:02 -03:00
session . drainWake = resolve ;
2026-08-23 22:13:52 -03:00
session . drainReject = reject ;
2026-08-23 18:29:02 -03:00
} ) ;
indicator . classList . add ( 'is-playing' ) ;
2026-08-23 22:13:52 -03:00
node . port . onmessage = function ( event ) {
const data = event . data || { } ;
if ( data . type === 'buffer' ) {
session . bufferedFrames = data . frames || 0 ;
if ( session . bufferedFrames < session . sampleRate && session . lowWaterWake ) {
const wake = session . lowWaterWake ;
session . lowWaterWake = null ;
wake ( ) ;
}
} else if ( data . type === 'drained' ) {
session . playbackEnded = true ;
const wake = session . drainWake ;
session . drainWake = null ;
if ( wake ) wake ( ) ;
} else if ( data . type === 'error' ) {
const reject = session . drainReject ;
session . drainReject = null ;
if ( reject ) reject ( new Error ( data . error || 'Streaming speech playback failed' ) ) ;
}
} ;
return session ;
}
async function playPcm ( asset , token ) {
const session = await ensurePcmPlayback ( asset , token ) ;
if ( ! session ) return ;
2026-08-23 18:29:02 -03:00
const reader = asset . response . body . getReader ( ) ;
let carry = null ;
2026-08-10 00:42:37 -03:00
try {
2026-08-23 18:29:02 -03:00
while ( active && token === generation && ! session . cancelled ) {
2026-08-23 22:13:52 -03:00
if ( session . bufferedFrames > asset . sampleRate * 2 ) {
2026-08-23 18:29:02 -03:00
await new Promise ( function ( resolve ) {
let completed = false ;
2026-08-23 22:13:52 -03:00
const wake = function ( ) { if ( completed ) return ; completed = true ; session . lowWaterWake = null ; resolve ( ) ; } ;
2026-08-23 18:29:02 -03:00
session . lowWaterWake = wake ;
window . setTimeout ( wake , 3000 ) ;
} ) ;
}
const result = await reader . read ( ) ;
if ( result . done ) break ;
let bytes = result . value ;
if ( carry !== null ) {
const joined = new Uint8Array ( bytes . length + 1 ) ;
joined [ 0 ] = carry ; joined . set ( bytes , 1 ) ; bytes = joined ; carry = null ;
}
if ( bytes . length % 2 ) { carry = bytes [ bytes . length - 1 ] ; bytes = bytes . slice ( 0 , - 1 ) ; }
if ( ! bytes . length ) continue ;
const copy = bytes . buffer . slice ( bytes . byteOffset , bytes . byteOffset + bytes . byteLength ) ;
2026-08-23 22:13:52 -03:00
session . bufferedFrames += bytes . byteLength / 2 ;
2026-08-23 18:29:02 -03:00
asset . started = true ;
2026-08-23 22:13:52 -03:00
session . node . port . postMessage ( { type : 'push' , samples : copy , sampleRate : asset . sampleRate } , [ copy ] ) ;
2026-08-10 00:42:37 -03:00
}
2026-08-23 18:29:02 -03:00
} finally {
2026-08-23 22:13:52 -03:00
if ( ( ! active || token !== generation || session . cancelled ) ) { try { await reader . cancel ( ) ; } catch ( _ ) { } }
2026-08-23 18:29:02 -03:00
session . controllers . delete ( asset . controller ) ;
}
}
2026-08-23 22:13:52 -03:00
async function drainPcmPlayback ( session , token ) {
if ( ! session || ! session . node || session . cancelled || ! active || token !== generation ) return ;
if ( session . playbackEnded ) return ;
session . node . port . postMessage ( { type : 'end' } ) ;
await session . drained ;
}
async function closePcmBeforeBlob ( session , token ) {
if ( ! session || ! session . node ) return ;
await drainPcmPlayback ( session , token ) ;
if ( session . node ) { try { session . node . disconnect ( ) ; } catch ( _ ) { } session . node = null ; }
if ( session . context ) { try { session . context . close ( ) ; } catch ( _ ) { } session . context = null ; }
session . gain = null ;
session . drained = null ;
indicator . classList . remove ( 'is-playing' ) ;
}
2026-08-23 18:29:02 -03:00
async function playPrepared ( asset , token ) {
2026-08-23 22:13:52 -03:00
if ( asset . kind !== 'pcm' ) {
if ( playbackSession && playbackSession . node ) await closePcmBeforeBlob ( playbackSession , token ) ;
return playBlob ( asset . blob , token ) ;
}
2026-08-23 18:29:02 -03:00
try {
return await playPcm ( asset , token ) ;
2026-08-10 00:42:37 -03:00
} catch ( error ) {
2026-08-23 18:29:02 -03:00
if ( asset . started ) throw error ;
streamingCapability . tts = null ;
2026-08-23 22:13:52 -03:00
if ( playbackSession && playbackSession . node ) await closePcmBeforeBlob ( playbackSession , token ) ;
return playBlob ( await fetchSpeech ( asset . chunk , asset . language , asset . turnId , token ) , token ) ;
2026-08-23 18:29:02 -03:00
}
}
function nextSpeechChunk ( turn ) {
if ( turn . cancelled ) return Promise . resolve ( null ) ;
if ( turn . queue . length ) return Promise . resolve ( turn . queue . shift ( ) ) ;
if ( turn . final ) return Promise . resolve ( null ) ;
return new Promise ( function ( resolve ) { turn . waiters . push ( resolve ) ; } ) ;
}
function enqueueSpeech ( turn , chunks ) {
chunks . forEach ( function ( chunk ) {
if ( ! chunk ) return ;
if ( turn . waiters . length ) turn . waiters . shift ( ) ( chunk ) ; else turn . queue . push ( chunk ) ;
} ) ;
}
function finishSpeechQueue ( turn ) {
turn . final = true ;
while ( turn . waiters . length && ! turn . queue . length ) turn . waiters . shift ( ) ( null ) ;
}
async function runSpeechQueue ( turn ) {
if ( turn . running ) return ;
turn . running = true ;
2026-08-23 22:13:52 -03:00
playbackSession = { turnId : turn . turnId , controllers : new Set ( ) , context : null , node : null , gain : null , sampleRate : 0 , bufferedFrames : 0 , drained : null , drainReject : null , playbackEnded : false , cancelled : false , lowWaterWake : null , drainWake : null , blobWake : null } ;
2026-08-23 18:29:02 -03:00
const session = playbackSession ;
try {
let chunk = await nextSpeechChunk ( turn ) ;
let current = chunk ? prepareSpeech ( chunk , turn . language , turn . turnId , turn . token ) : null ;
while ( current ) {
const asset = await current ;
// At most one later synthesis request exists while this asset plays.
// If streaming has not produced it yet, the waiter starts it the
// instant a punctuation-safe chunk arrives.
const next = nextSpeechChunk ( turn ) ;
const nextPrepared = next . then ( function ( value ) {
return value ? prepareSpeech ( value , turn . language , turn . turnId , turn . token ) : null ;
} ) ;
2026-08-23 22:13:52 -03:00
// Barge-in can abort both the playing request and its one-ahead request.
// Observe the latter even when the cancelled current turn returns first.
nextPrepared . catch ( function ( ) { } ) ;
2026-08-23 18:29:02 -03:00
await playPrepared ( asset , turn . token ) ;
if ( ! active || turn . token !== generation || turn . cancelled || session . cancelled ) return ;
current = await nextPrepared ;
}
2026-08-23 22:13:52 -03:00
await drainPcmPlayback ( session , turn . token ) ;
2026-08-23 18:29:02 -03:00
if ( turn . final && active && turn . token === generation && ! turn . cancelled ) restartSoon ( turn . token , 300 ) ;
} catch ( error ) {
if ( active && turn . token === generation && ! turn . cancelled ) {
2026-08-20 23:13:51 +00:00
const message = errorMessage ( error , 'Local speech is unavailable' ) ;
setState ( 'error' , message ) ;
toast ( message ) ;
2026-08-23 18:29:02 -03:00
restartSoon ( turn . token , 500 ) ;
}
} finally {
if ( playbackSession === session ) stopPlayback ( ) ;
if ( speechTurn === turn ) speechTurn = null ;
}
}
function currentAssistantText ( ) {
const rows = assistantRows ( ) ;
if ( ! rows . length ) return '' ;
const row = rows [ rows . length - 1 ] ;
const text = readAssistantRow ( row ) ;
if ( assistantBaseline && row === assistantBaseline . row && text === assistantBaseline . text ) return '' ;
if ( assistantBaseline && rows . length < assistantBaseline . count ) return '' ;
return cleanForSpeech ( text ) ;
}
function ensureSpeechTurn ( token ) {
if ( speechTurn ) return speechTurn ;
speechTurn = {
token : token ,
turnId : thinkingTurnId || String ( token ) + '-' + String ( ++ turnSequence ) ,
language : takeSttLanguage ( token ) ,
sourceText : '' ,
consumed : 0 ,
first : true ,
queue : [ ] ,
waiters : [ ] ,
final : false ,
cancelled : false ,
running : false ,
} ;
return speechTurn ;
}
function pumpAssistantResponse ( token , isFinal ) {
if ( ! active || token !== generation || ( state !== 'thinking' && state !== 'speaking' ) ) return ;
const currentSession = ( typeof S !== 'undefined' && S . session ) ? S . session . session _id : null ;
if ( thinkingSession && currentSession && thinkingSession !== currentSession ) {
thinkingSession = null ;
thinkingTurnId = '' ;
clearSttLanguage ( ) ;
stopResponseObserver ( ) ;
2026-08-23 22:13:52 -03:00
cancelThinkingCues ( ) ;
2026-08-23 18:29:02 -03:00
cancelSpeechTurn ( ) ;
stopPlayback ( ) ;
restartSoon ( token , 250 ) ;
return ;
}
const text = currentAssistantText ( ) ;
if ( ! text ) {
2026-08-23 22:13:52 -03:00
if ( isFinal ) { thinkingSession = null ; thinkingTurnId = '' ; clearSttLanguage ( ) ; stopResponseObserver ( ) ; cancelThinkingCues ( ) ; restartSoon ( token , 250 ) ; }
2026-08-23 18:29:02 -03:00
return ;
}
2026-08-23 22:13:52 -03:00
// Any answer text, even a not-yet-speakable partial clause, owns the audio
// timeline from this point forward. A cue must never overlap or become part
// of the answer queue.
cancelThinkingCues ( ) ;
2026-08-23 18:29:02 -03:00
const turn = ensureSpeechTurn ( token ) ;
if ( text . length < turn . sourceText . length || ! text . startsWith ( turn . sourceText ) ) {
// Renderers can revise the still-unspoken tail. Already-spoken text is
// immutable. On the completion callback, advance from the old spoken
// offset and close the queue even when the renderer rewrote an earlier
// span; replaying the revised prefix would be more disruptive than
// preserving the already-heard words.
if ( turn . consumed > 0 ) {
if ( ! isFinal ) return ;
thinkingSession = null ;
thinkingTurnId = '' ;
stopResponseObserver ( ) ;
const revisedTail = text . slice ( Math . min ( turn . consumed , text . length ) ) . trim ( ) ;
if ( revisedTail ) {
const revised = adaptiveChunks ( revisedTail , true , false ) ;
enqueueSpeech ( turn , revised . chunks . length ? revised . chunks : [ revisedTail ] ) ;
turn . consumed = text . length ;
}
finishSpeechQueue ( turn ) ;
return ;
2026-08-20 23:13:51 +00:00
}
2026-08-23 18:29:02 -03:00
turn . sourceText = '' ;
}
turn . sourceText = text ;
const remaining = text . slice ( turn . consumed ) . trimStart ( ) ;
const skipped = text . slice ( turn . consumed ) . length - remaining . length ;
const extracted = adaptiveChunks ( remaining , ! ! isFinal , turn . first ) ;
if ( extracted . chunks . length ) {
turn . first = false ;
turn . consumed += skipped + extracted . consumed ;
enqueueSpeech ( turn , extracted . chunks ) ;
setState ( 'speaking' ) ;
runSpeechQueue ( turn ) ;
}
if ( isFinal ) {
thinkingSession = null ;
thinkingTurnId = '' ;
stopResponseObserver ( ) ;
const tail = text . slice ( turn . consumed ) . trim ( ) ;
if ( tail ) { enqueueSpeech ( turn , [ tail ] ) ; turn . consumed = text . length ; }
finishSpeechQueue ( turn ) ;
}
}
function startResponseObserver ( token ) {
stopResponseObserver ( ) ;
// A bounded poll is deliberately used in addition to the upstream
// completion callback. It sees data-raw-text while SSE is still appending,
// so the first complete sentence can reach Piper before generation ends.
responsePollTimer = window . setInterval ( function ( ) { pumpAssistantResponse ( token , false ) ; } , 75 ) ;
}
function speakResponse ( token ) {
pumpAssistantResponse ( token , true ) ;
}
function validStreamingCapability ( payload ) {
2026-08-23 22:13:52 -03:00
if ( ! payload || typeof payload !== 'object' ) return { tts : null , stt : null , preflight : null } ;
2026-08-23 18:29:02 -03:00
const tts = payload . tts && payload . tts . available === true && payload . tts . transport === 'http' && payload . tts . format === 'pcm_s16le' ? payload . tts : null ;
const stt = payload . stt && payload . stt . available === true && payload . stt . transport === 'websocket' && payload . stt . format === 'pcm_s16le' && payload . stt . sample _rate === 16000 ? payload . stt : null ;
2026-08-23 22:13:52 -03:00
const preflight = payload . preflight && payload . preflight . available === true && payload . preflight . path === VOICE _PREFLIGHT _URL && payload . preflight . advisory === true ? payload . preflight : null ;
return { tts : tts , stt : stt , preflight : preflight } ;
2026-08-23 18:29:02 -03:00
}
async function discoverStreamingCapability ( ) {
try {
const response = await fetch ( STREAMING _CAPABILITY _URL , { cache : 'no-store' } ) ;
if ( ! response . ok ) return ;
streamingCapability = validStreamingCapability ( await response . json ( ) ) ;
} catch ( _ ) {
2026-08-23 22:13:52 -03:00
streamingCapability = { tts : null , stt : null , preflight : null } ;
2026-08-10 00:42:37 -03:00
}
}
function activate ( ) {
generation += 1 ;
const token = generation ;
active = true ;
2026-08-20 23:13:51 +00:00
clearErrorTimer ( ) ;
2026-08-21 11:52:49 +00:00
clearSttLanguage ( ) ;
2026-08-10 00:42:37 -03:00
modeBtn . classList . add ( 'active' ) ;
toast ( 'Hands-free private voice mode on' ) ;
if ( typeof window . stopTTS === 'function' ) window . stopTTS ( ) ;
if ( typeof S !== 'undefined' && S . busy ) { setState ( 'thinking' ) ; return ; }
startListening ( token ) ;
}
function onVoiceClick ( event ) {
if ( ! ready ) return ;
event . preventDefault ( ) ;
event . stopImmediatePropagation ( ) ;
if ( active ) deactivate ( true ) ; else activate ( ) ;
}
async function initialize ( ) {
try {
const response = await fetch ( '/api/transcribe/capability' , { cache : 'no-store' } ) ;
const capability = await response . json ( ) . catch ( function ( ) { return { } ; } ) ;
if ( ! response . ok || ! capability . available || capability . provider !== 'local_command' ) return ;
ready = true ;
2026-08-23 18:29:02 -03:00
discoverStreamingCapability ( ) ;
2026-08-10 00:42:37 -03:00
if ( localStorage . getItem ( 'hermes-atlas-voice-initialized' ) !== '1' ) {
localStorage . setItem ( 'hermes-atlas-voice-initialized' , '1' ) ;
localStorage . setItem ( 'hermes-voice-mode-button' , 'true' ) ;
localStorage . setItem ( 'hermes-tts-engine' , 'atlas' ) ;
localStorage . setItem ( 'hermes-tts-enabled' , 'true' ) ;
2026-08-23 16:50:48 -03:00
}
// Move existing installations to the lower-latency silence window once;
// subsequent user changes remain untouched.
if ( localStorage . getItem ( 'hermes-atlas-voice-latency-v2' ) !== '1' ) {
localStorage . setItem ( 'hermes-atlas-voice-latency-v2' , '1' ) ;
localStorage . setItem ( 'hermes-voice-silence-ms' , '1100' ) ;
2026-08-10 00:42:37 -03:00
}
const selector = document . getElementById ( 'settingsTtsEngine' ) ;
if ( selector && ! selector . querySelector ( 'option[value="atlas"]' ) ) {
const option = document . createElement ( 'option' ) ;
option . value = 'atlas' ;
option . textContent = 'Atlas Jetson (private)' ;
selector . insertBefore ( option , selector . firstChild ) ;
}
modeBtn . style . display = localStorage . getItem ( 'hermes-voice-mode-button' ) === 'false' ? 'none' : '' ;
modeBtn . addEventListener ( 'click' , onVoiceClick , true ) ;
window . _applyVoiceModePref = function ( ) {
if ( typeof originalApplyPreference === 'function' ) originalApplyPreference ( ) ;
if ( ready ) {
const enabled = localStorage . getItem ( 'hermes-voice-mode-button' ) !== 'false' ;
modeBtn . style . display = enabled ? '' : 'none' ;
if ( ! enabled && active ) deactivate ( false ) ;
}
} ;
window . autoReadLastAssistant = function ( ) {
2026-08-23 22:13:52 -03:00
if ( active ) {
if ( suppressAutoRead || bargeCancelPromise || state === 'listening' || state === 'transcribing' ) return ;
if ( state === 'thinking' || state === 'speaking' ) { speakResponse ( generation ) ; return ; }
}
2026-08-10 00:42:37 -03:00
if ( typeof originalAutoRead === 'function' ) originalAutoRead . apply ( this , arguments ) ;
} ;
window . _voiceModeActive = function ( ) { return active ; } ;
window . _voiceModeDeactivate = function ( ) { deactivate ( false ) ; } ;
window . _voiceModeImmediateSend = function ( ) {
if ( active && recorder && recorder . state !== 'inactive' ) recorder . stop ( ) ;
} ;
} catch ( _ ) {
// The upstream browser voice implementation remains available as fallback.
}
}
initialize ( ) ;
} ) ( ) ;