diff --git a/dockerfiles/hermes-webui-atlas-patch.py b/dockerfiles/hermes-webui-atlas-patch.py index 5b9ff14c..97b83c10 100644 --- a/dockerfiles/hermes-webui-atlas-patch.py +++ b/dockerfiles/hermes-webui-atlas-patch.py @@ -750,6 +750,73 @@ replace_exact( return _handle_tts(handler, parsed) ''', ) +# The chat router's session-continuity poller renders whatever /api/session +# answers. Its 409 branch ("This session is unavailable to this account.") +# is only ever correct when the root-profile alias set is actually known — +# but list_profiles_api() is a hermes_cli subprocess call that fails +# transiently (cold pod after a roll, load spikes), and the pinned +# _is_root_profile() treats that failure as "not a root alias", flipping +# _profiles_match() to a false mismatch and painting the bogus banner over +# the input bar until the next successful listing. Two minimal grafts: +# answer alias checks from the last known alias set on listing failure, and +# never claim a default-vs-named mismatch while the alias set is unconfirmed. +profiles = ROOT / "api/profiles.py" +replace_exact( + profiles, + ''' except Exception: + logger.debug("Failed to list profiles for root-profile lookup", exc_info=True) + return False +''', + ''' except Exception: + logger.debug("Failed to list profiles for root-profile lookup", exc_info=True) + # Atlas voice patch: a transient listing failure must not deny a name + # that was already confirmed as a root alias — answer from the last + # known alias set instead of failing the alias outright. + with _root_profile_name_cache_lock: + return name in _root_profile_name_cache +''', +) +replace_exact( + profiles, + "def _is_root_profile(name: str) -> bool:\n", + '''def _root_profile_names_confirmed() -> bool: + """True once list_profiles_api() has successfully populated the alias set. + + Atlas voice patch: lets _profiles_match() distinguish "these profiles are + definitely different" from "the root-alias equivalence could not be + checked yet" (cold cache right after a pod roll, or a failing hermes_cli + listing), which previously produced transient /api/session 409s. + """ + with _root_profile_name_cache_lock: + return _root_profile_name_cache_loaded + + +def _is_root_profile(name: str) -> bool: +''', +) +replace_exact( + profiles, + ''' # Cross-alias the renamed root. + if _is_root_profile(row) and _is_root_profile(active): + return True + return False +''', + ''' # Cross-alias the renamed root. + if _is_root_profile(row) and _is_root_profile(active): + return True + # Atlas voice patch: while the root alias set is unconfirmed (cold cache + # after a pod roll, hermes_cli listing failure) a pair involving the + # 'default' alias cannot be *proven* mismatched — the named side may be + # the renamed root. Fail open for that pair only: a mismatch between two + # named profiles is still denied, and exact scoping resumes with the + # first successful listing. This removes the transient /api/session 409 + # that rendered a false "unavailable to this account" banner. + if "default" in (row, active) and not _root_profile_names_confirmed(): + return True + return False +''', +) + marker = " # ── ElevenLabs TTS ──────────────────────────────────────────────────\n" atlas = ''' # ── Atlas private Jetson TTS ───────────────────────────────────────── if engine == "atlas": diff --git a/dockerfiles/hermes-webui-atlas-voice.css b/dockerfiles/hermes-webui-atlas-voice.css index b587a124..34a96e87 100644 --- a/dockerfiles/hermes-webui-atlas-voice.css +++ b/dockerfiles/hermes-webui-atlas-voice.css @@ -463,13 +463,36 @@ flex-direction: column; gap: 10px; width: min(100%, 40rem); - max-height: 30vh; - overflow: hidden; + min-height: 0; text-align: center; } +/* Each caption is its own bounded scroll region: the user transcript and the + reply overflow independently, scroll by touch inside the overlay without + moving the page behind (overscroll-behavior: contain), stay keyboard + scrollable via tabindex, and fade at the top edge so clipped history reads + as scrollable. Auto-follow of the streaming tail lives in atlas-voice.js + (data-follow, per region). */ +.voice-conversation-caption-user, +.voice-conversation-caption-assistant { + overflow-y: auto; + touch-action: pan-y; + -webkit-overflow-scrolling: touch; + overscroll-behavior: contain; + scrollbar-width: thin; + -webkit-mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 100%); + mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 100%); +} + +.voice-conversation-caption-user:focus-visible, +.voice-conversation-caption-assistant:focus-visible { + outline: 2px solid rgb(var(--voice-accent)); + outline-offset: 2px; +} + .voice-conversation-caption-user { margin: 0; + max-height: 22vh; font-size: 14px; line-height: 1.5; color: rgba(178, 196, 214, 0.88); @@ -478,6 +501,7 @@ .voice-conversation-caption-assistant { margin: 0; + max-height: 38vh; font-size: 16px; line-height: 1.6; color: rgba(240, 247, 255, 0.95); @@ -550,7 +574,8 @@ @media (max-width: 640px) { .voice-conversation { gap: 18px; } - .voice-conversation-captions { max-height: 34vh; } + .voice-conversation-caption-user { max-height: 24vh; } + .voice-conversation-caption-assistant { max-height: 34vh; } } /* Reduced motion: static orb states — the accent color and the state caption diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index fed8ffc1..94bf648e 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -112,6 +112,11 @@ // bound to that turn's generation token and consumed exactly once. let sttLanguage=''; let sttLanguageToken=-1; + // Sticky per-hands-free-session language: set by the private Whisper + // detection of the user's own speech (or a strong reply-text signal on a + // detection-less turn), consumed as the NEXT streaming STT session's bias + // and as the thinking-cue locale. Cleared on activate/deactivate. + let sessionLanguage=''; const TTS_LANGUAGES=['en','ru','es']; const originalAutoRead=window.autoReadLastAssistant; const originalApplyPreference=window._applyVoiceModePref; @@ -138,6 +143,37 @@ return language; } + const SPANISH_ORTHOGRAPHY=/[áéíóúñü¡¿]/i; + const SPANISH_STOPWORDS=/\b(?:el|la|los|las|un|una|es|está|qué|para|por|con|pero|como|más|sí|gracias|hola|puedo|también|muy|este|esta|todo|bien)\b/g; + + function strongReplyLanguage(text){ + // Script-level certainty only: Cyrillic text is Russian; Spanish + // orthography (accents, ñ, inverted punctuation) is Spanish. Plain-ASCII + // text yields no signal, so an English reply never flips a trusted + // STT-detected voice. + const sample=String(text||'').slice(0,400); + if(/[Ѐ-ӿ]/.test(sample)) return 'ru'; + if(SPANISH_ORTHOGRAPHY.test(sample)) return 'es'; + return ''; + } + + function detectReplyLanguage(text){ + // Lightweight reply-language heuristic for turns without a trusted STT + // detection: script evidence first, then Spanish stopword density (an + // accent-free Spanish sentence still routes to the Spanish voice). + // Returns '' for English/unknown, which the private TTS service resolves + // to its own English default voice. + const strong=strongReplyLanguage(text); + if(strong) return strong; + const sample=String(text||'').slice(0,400).toLowerCase(); + const words=sample.split(/\s+/).filter(Boolean); + if(words.length>=4){ + const matches=(sample.match(SPANISH_STOPWORDS)||[]).length; + if(matches>=2&&matches/words.length>=0.12) return 'es'; + } + return ''; + } + function toast(message){ if(typeof window.showToast==='function') window.showToast(message,3000); } @@ -194,12 +230,32 @@ conversation.stateEl.textContent=customLabel||STATE_LABELS[next]||''; } + function updateCaptionRegion(element,text,limit){ + // Captions are bounded scrollable regions: keep following the streaming + // tail unless the user scrolled up inside this region (data-follow='0', + // maintained by the scroll listener installed at overlay build time). + const value=String(text||'').slice(-limit); + element.textContent=value; + if(!value&&element.dataset) element.dataset.follow='1'; + if((!element.dataset||element.dataset.follow!=='0')&&typeof element.scrollHeight==='number'){ + try{element.scrollTop=element.scrollHeight;}catch(_){ } + } + } + + function attachCaptionScroll(element){ + if(!element.addEventListener) return; + element.addEventListener('scroll',function(){ + const gap=(element.scrollHeight||0)-(element.scrollTop||0)-(element.clientHeight||0); + if(element.dataset) element.dataset.follow=gap<=24?'1':'0'; + }); + } + function setConversationUserCaption(text){ - if(conversation) conversation.userCaption.textContent=String(text||'').slice(-600); + if(conversation) updateCaptionRegion(conversation.userCaption,text,4000); } function setConversationAssistantCaption(text){ - if(conversation) conversation.assistantCaption.textContent=String(text||'').slice(-900); + if(conversation) updateCaptionRegion(conversation.assistantCaption,text,9000); } function setConversationPlaying(playing){ @@ -261,8 +317,13 @@ orb.appendChild(conversationNode('span','voice-conversation-orb-ring')); const stateEl=conversationNode('div','voice-conversation-state',{'aria-hidden':'true'}); const captions=conversationNode('div','voice-conversation-captions',{'aria-live':'polite'}); - const userCaption=conversationNode('p','voice-conversation-caption-user'); - const assistantCaption=conversationNode('p','voice-conversation-caption-assistant'); + // Each caption is its own bounded, independently scrollable region + // (keyboard focusable, touch scrollable); streaming keeps following the + // tail until the user scrolls up inside that region. + const userCaption=conversationNode('p','voice-conversation-caption-user',{tabindex:'0'}); + const assistantCaption=conversationNode('p','voice-conversation-caption-assistant',{tabindex:'0'}); + attachCaptionScroll(userCaption); + attachCaptionScroll(assistantCaption); captions.appendChild(userCaption); captions.appendChild(assistantCaption); const controls=conversationNode('div','voice-conversation-controls'); @@ -328,6 +389,7 @@ function cancelSpeechTurn(){ if(!speechTurn) return; + cancelSpeakingIdleFallback(speechTurn); speechTurn.cancelled=true; speechTurn.final=true; while(speechTurn.waiters.length) speechTurn.waiters.shift()(null); @@ -453,6 +515,18 @@ if(ducked) indicator.classList.add('is-ducked'); else indicator.classList.remove('is-ducked'); } + function playbackAudible(){ + // True only while audio is actually sounding. A streaming PCM session + // keeps its worklet node alive across sentence chunks and between the + // speech segments of one turn; the buffered frames the worklet reports + // decide whether it is audible right now. + if(currentAudio) return true; + if(thinkingCue&&thinkingCue.node) return true; + const session=playbackSession; + if(session&&session.node) return session.bufferedFrames>0&&!session.playbackEnded; + return indicator.classList.contains('is-playing'); + } + function disposeBargeResources(monitor,keepCapture){ if(!monitor) return; monitor.cancelled=true; @@ -570,6 +644,7 @@ clearErrorTimer(); removeConversationOverlay(); clearSttLanguage(); + sessionLanguage=''; suppressAutoRead=false; pendingStitch=null; lastSentTranscript=null; @@ -595,25 +670,117 @@ },delay||500); } + function resyncCapture(token,statusLabel){ + // Full capture-turn resync: cancel any streaming STT session, drop the + // lookback/pending buffers and start a fresh capture turn on the hot + // microphone. Used after an errored turn and by barge-in edge cases + // where the previous stream state is no longer trustworthy. + if(!active||token!==generation) return; + stopCapture(); + startListening(token); + setState('listening',statusLabel); + } + + function handleAssistantResponseError(token){ + // An error/system envelope (cancellation notice, provider failure) is a + // transcript artifact, not a reply: never feed it to TTS or the reply + // caption, show a brief non-spoken state instead, and resynchronize + // capture so the next utterance starts a clean streaming turn. + thinkingSession=null; + thinkingTurnId=''; + clearSttLanguage(); + stopResponseObserver(); + cancelThinkingCues(); + cancelSpeechTurn(); + stopPlayback(); + pendingStitch=null; + lastSentTranscript=null; + clearBargeCancellation(); + setConversationAssistantCaption(''); + resyncCapture(token,'Something went wrong — listening'); + } + function assistantRows(){ return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]'); } - function readAssistantRow(row){ - if(!row) return ''; - if(row.dataset&&typeof row.dataset.rawText==='string') return row.dataset.rawText; - return typeof row.textContent==='string'?row.textContent:''; + function assistantTurnOf(row){ + // Resolve a matched node to its whole turn container so multi-segment + // turns (interim messages around tool calls) are read as one response. + if(row&&typeof row.closest==='function'){ + const turn=row.closest('.msg-row[data-role="assistant"]'); + if(turn) return turn; + } + return row; + } + + function segmentIsHidden(segment){ + if(!segment) return true; + if(segment.hidden===true) return true; + return !!(typeof segment.getAttribute==='function'&&segment.getAttribute('aria-hidden')==='true'); + } + + function segmentIsError(segment){ + // Error/system envelopes: the renderer stamps data-error="1" on segments + // matching its error patterns, and provider errors and cancellation + // notices carry a .provider-error-details block inside the body. + if(!segment) return false; + if(segment.dataset&&segment.dataset.error==='1') return true; + return !!(typeof segment.querySelector==='function'&&segment.querySelector('.provider-error-details')); + } + + function readSegmentBody(segment){ + if(!segment) return ''; + if(segment.dataset&&typeof segment.dataset.rawText==='string') return segment.dataset.rawText; + // Scrape only message BODY text — never the avatar letter, author name, + // "Processed Ns" worklog chips or footer controls that share the row. + if(typeof segment.querySelectorAll==='function'){ + const bodies=segment.querySelectorAll('.msg-body'); + if(bodies&&bodies.length){ + return Array.prototype.map.call(bodies,function(body){return body.textContent||'';}).join('\n'); + } + } + return typeof segment.textContent==='string'?segment.textContent:''; + } + + function readAssistantTurn(turn){ + // {text, error}: body-only text of every visible answer segment in the + // turn, plus whether any segment is an error/system envelope. + if(!turn) return {text:'',error:false}; + let error=false; + const parts=[]; + const segments=(typeof turn.querySelectorAll==='function')?turn.querySelectorAll('.assistant-segment'):null; + if(segments&&segments.length){ + Array.prototype.forEach.call(segments,function(segment){ + if(segmentIsHidden(segment)) return; + if(segmentIsError(segment)){error=true;return;} + const value=readSegmentBody(segment); + if(value&&value.trim()) parts.push(value.trim()); + }); + }else if(segmentIsError(turn)){ + error=true; + }else{ + const value=readSegmentBody(turn); + if(value&&value.trim()) parts.push(value.trim()); + } + return {text:cleanForSpeech(parts.join('\n\n')),error:error}; } function rememberAssistantBaseline(){ const rows=assistantRows(); - const row=rows.length?rows[rows.length-1]:null; - assistantBaseline={row:row,text:readAssistantRow(row),count:rows.length}; + const turn=rows.length?assistantTurnOf(rows[rows.length-1]):null; + assistantBaseline={row:turn,text:readAssistantTurn(turn).text,count:rows.length}; } async function settleBargeCancellation(token){ const cancellation=bargeCancelPromise; if(!cancellation) return true; + if(!cancellation.streamId){ + // Nothing was actually in flight to cancel: a stale busy flag left by + // an errored turn must never hold the send in a dead settle wait. + if(bargeCancelPromise===cancellation) bargeCancelPromise=null; + return true; + } try{await cancellation.promise;}catch(_){ } const deadline=Date.now()+10000; while(active&&token===generation&&Date.now()120?tail.slice(tail.length-120):tail; + return '\n[voice interruption: you were cut off after "'+bounded+'"]'; } async function sendTranscript(transcript,token,language,turnId){ @@ -662,7 +844,7 @@ // restated as one stitched message, so the model answers the full thought. const stitch=(pendingStitch&&pendingStitch.text&&(Date.now()-pendingStitch.at)<=STITCH_WINDOW_MS)?pendingStitch:null; pendingStitch=null; - composer.value=stitch?stitch.text+' '+text:text; + composer.value=stitch?stitch.text+' '+text+voiceCutMarker(stitch.cut):text; if(typeof window.autoResize==='function') window.autoResize(); setConversationUserCaption(composer.value); setConversationAssistantCaption(''); @@ -675,19 +857,34 @@ if(!cancellationSettled){ pendingStitch=stitch; toast('The previous response did not stop. Please repeat your interruption.'); - restartSoon(token,250); + // The stream state is now suspect: rebuild the capture turn instead of + // resuming a session whose server-side epoch may be stale. + resyncCapture(token); return; } thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null; thinkingTurnId=turnId||captureTurnId||String(token)+'-'+String(++turnSequence); rememberAssistantBaseline(); rememberSttLanguage(language,token); + // The user's detected speech language is the sticky hands-free session + // language: it biases the next streaming STT session and localizes the + // thinking cues until the user audibly switches again. + if(language){ + const switched=language!==sessionLanguage; + sessionLanguage=language; + // Continuous capture already opened the next session (with the + // previous bias) before this turn's detection arrived: restart it with + // the switched language, but only while it has heard nothing yet. + if(switched&&captureActive&&streamingStt&&typeof streamingStt.latestPartial==='function'&&!streamingStt.latestPartial()){ + startListening(generation,{preserveDisplay:true}); + } + } if(typeof window.send==='function'){ lastSentTranscript={text:composer.value,token:token}; window.send(); suppressAutoRead=false; startResponseObserver(token); - scheduleThinkingCues(token,language,thinkingTurnId); + scheduleThinkingCues(token,language||sessionLanguage,thinkingTurnId); } } @@ -798,6 +995,7 @@ let workletReady=false; let partialRevision=-1; let lastPreflightText=''; + let lastPartialText=''; let queue=[]; let queuedBytes=0; let flushTimer=null; @@ -862,7 +1060,9 @@ return null; } socket.onopen=function(){ - sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:'auto'}); + // Bias recognition toward the session's sticky language the moment the + // user has audibly switched; 'auto' remains the cold-start default. + sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:sessionLanguage||'auto'}); flush(); }; socket.onmessage=function(event){ @@ -877,6 +1077,10 @@ const stable=String(payload.stable_transcript||'').trim(); const provisional=String(payload.transcript||'').trim(); const visible=stable||provisional; + // The newest rolling partial also feeds the dynamic endpoint below, + // so it is tracked regardless of the display-only gating that keeps + // the label and captions quiet outside the listening state. + if(visible) lastPartialText=visible; if(visible&&active&&captureTurnId===turnId&&state==='listening'){ const preview=visible.length>72?visible.slice(0,69)+'…':visible; label.textContent='Listening · '+preview+(stable?'':' · provisional'); @@ -904,6 +1108,7 @@ return { finalPromise:finalPromise, + latestPartial:function(){return lastPartialText;}, setWorkletReady:function(value){workletReady=value;}, push:function(samples){ if(cancelled||committed||!workletReady) return; @@ -1149,7 +1354,7 @@ for(let index=0;index=3 words, or terminal punctuation), endpoint at the base window + // instead so short commands are not delayed by the clipping guard — + // the long hold remains only for 1-2 word partials. + const partialText=streamingStt&&streamingStt.latestPartial?streamingStt.latestPartial():''; + const partialWords=partialText?partialText.split(/\s+/).filter(Boolean).length:0; + const partialComplete=partialWords>=3||(partialWords>0&&/[.!?…]["')\]}]*$/.test(partialText)); + const endpointSilenceMs=(speechMs=endpointSilenceMs; const timedOut=now-started>=90000; const idle=(!heardSpeech)&&(now-started)>=20000; @@ -1775,7 +1987,7 @@ streamingCapability.tts=null; } } - return {kind:'blob',blob:await fetchSpeech(chunk,language,turnId,token)}; + return {kind:'blob',chunk:chunk,blob:await fetchSpeech(chunk,language,turnId,token)}; } async function ensurePcmPlayback(asset,token){ @@ -1923,6 +2135,25 @@ while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null); } + function scheduleSpeakingIdleFallback(turn,session){ + // Interim-message turns speak in cycles (speak → tools → speak): once no + // further chunk is queued and the turn is not final, fall back to + // Thinking after the buffered audio has played out. The observer flips + // the state back to Speaking when the next segment yields a chunk. + cancelSpeakingIdleFallback(turn); + const bufferedMs=session&&session.sampleRate?Math.ceil(((session.bufferedFrames||0)/session.sampleRate)*1000):0; + turn.idleTimer=window.setTimeout(function(){ + turn.idleTimer=null; + if(!active||turn.token!==generation||turn.cancelled||speechTurn!==turn) return; + if(turn.queue.length||turn.final) return; + if(state==='speaking') setState('thinking'); + },bufferedMs+250); + } + + function cancelSpeakingIdleFallback(turn){ + if(turn&&turn.idleTimer){window.clearTimeout(turn.idleTimer);turn.idleTimer=null;} + } + async function runSpeechQueue(turn){ if(turn.running) return; turn.running=true; @@ -1943,9 +2174,15 @@ // Barge-in can abort both the playing request and its one-ahead request. // Observe the latter even when the cancelled current turn returns first. nextPrepared.catch(function(){ }); + turn.speakingChunk=asset.chunk||''; await playPrepared(asset,turn.token); + turn.lastSpokenChunk=asset.chunk||turn.lastSpokenChunk; + turn.speakingChunk=''; if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return; + if(!turn.queue.length&&!turn.final) scheduleSpeakingIdleFallback(turn,session); current=await nextPrepared; + cancelSpeakingIdleFallback(turn); + if(current) setState('speaking'); } await drainPcmPlayback(session,turn.token); if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300); @@ -1962,23 +2199,34 @@ } } - function currentAssistantText(){ + function collectAssistantResponse(){ const rows=assistantRows(); - if(!rows.length) return ''; - const row=rows[rows.length-1]; - const text=readAssistantRow(row); - if(assistantBaseline&&row===assistantBaseline.row&&text===assistantBaseline.text) return ''; - if(assistantBaseline&&rows.length