hermes(voice): round-3 fixes, language routing, session-toast root cause

- Captions read message bodies only (the scraper was concatenating
  avatar, author and worklog chips); multi-segment interim turns are
  now speakable and drive clean speak-to-thinking-to-speak cycles when
  playback drains mid-turn.
- Dynamic endpointing: complete-looking partials (3+ words or terminal
  punctuation) endpoint at the base window; the long hold remains only
  for one-two-word fragments. A stale-busy 10s settle wait on every
  post-error send is gone.
- Both overlay captions are bounded, touch-scrollable regions with
  follow-tail; caps raised for long turns.
- Error envelopes are never spoken or captioned; errored turns run
  resyncCapture (fresh STT session on the hot mic).
- Workspace toggle now lives in the sidebar rail (floating button only
  below the rail breakpoint).
- False 'session unavailable' toast root-caused: the router continuity
  guard shows it on a 409 that fired when a transient profile-listing
  failure failed closed into a fake cross-profile mismatch; the patcher
  now answers from the alias cache and never claims a default-vs-named
  mismatch while aliases are unconfirmed.
- Barge-in sends carry a one-line cut-point marker with the last spoken
  sentence; visible-history truncation judged infeasible client-side.
- Language switching works end-to-end: sticky per-session STT language
  hint (restarting an unused next session on switch), reply voice from
  script evidence, STT detection, then stopword heuristic; cues and WAV
  fallback share the turn language.
- Legacy CI guards: node skip for the DOM probe, ffmpeg/codec skips for
  the container-fallback test. 272 voice-lane tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
This commit is contained in:
jenkins 2026-08-24 13:58:35 -03:00
parent c0a9c92ee4
commit d041f1d1ee
14 changed files with 916 additions and 43 deletions

View File

@ -750,6 +750,73 @@ replace_exact(
return _handle_tts(handler, parsed) return _handle_tts(handler, parsed)
''', ''',
) )
# The chat router's session-continuity poller renders whatever /api/session
# answers. Its 409 branch ("This session is unavailable to this account.")
# is only ever correct when the root-profile alias set is actually known —
# but list_profiles_api() is a hermes_cli subprocess call that fails
# transiently (cold pod after a roll, load spikes), and the pinned
# _is_root_profile() treats that failure as "not a root alias", flipping
# _profiles_match() to a false mismatch and painting the bogus banner over
# the input bar until the next successful listing. Two minimal grafts:
# answer alias checks from the last known alias set on listing failure, and
# never claim a default-vs-named mismatch while the alias set is unconfirmed.
profiles = ROOT / "api/profiles.py"
replace_exact(
profiles,
''' except Exception:
logger.debug("Failed to list profiles for root-profile lookup", exc_info=True)
return False
''',
''' except Exception:
logger.debug("Failed to list profiles for root-profile lookup", exc_info=True)
# Atlas voice patch: a transient listing failure must not deny a name
# that was already confirmed as a root alias — answer from the last
# known alias set instead of failing the alias outright.
with _root_profile_name_cache_lock:
return name in _root_profile_name_cache
''',
)
replace_exact(
profiles,
"def _is_root_profile(name: str) -> bool:\n",
'''def _root_profile_names_confirmed() -> bool:
"""True once list_profiles_api() has successfully populated the alias set.
Atlas voice patch: lets _profiles_match() distinguish "these profiles are
definitely different" from "the root-alias equivalence could not be
checked yet" (cold cache right after a pod roll, or a failing hermes_cli
listing), which previously produced transient /api/session 409s.
"""
with _root_profile_name_cache_lock:
return _root_profile_name_cache_loaded
def _is_root_profile(name: str) -> bool:
''',
)
replace_exact(
profiles,
''' # Cross-alias the renamed root.
if _is_root_profile(row) and _is_root_profile(active):
return True
return False
''',
''' # Cross-alias the renamed root.
if _is_root_profile(row) and _is_root_profile(active):
return True
# Atlas voice patch: while the root alias set is unconfirmed (cold cache
# after a pod roll, hermes_cli listing failure) a pair involving the
# 'default' alias cannot be *proven* mismatched — the named side may be
# the renamed root. Fail open for that pair only: a mismatch between two
# named profiles is still denied, and exact scoping resumes with the
# first successful listing. This removes the transient /api/session 409
# that rendered a false "unavailable to this account" banner.
if "default" in (row, active) and not _root_profile_names_confirmed():
return True
return False
''',
)
marker = " # ── ElevenLabs TTS ──────────────────────────────────────────────────\n" marker = " # ── ElevenLabs TTS ──────────────────────────────────────────────────\n"
atlas = ''' # ── Atlas private Jetson TTS ───────────────────────────────────────── atlas = ''' # ── Atlas private Jetson TTS ─────────────────────────────────────────
if engine == "atlas": if engine == "atlas":

View File

@ -463,13 +463,36 @@
flex-direction: column; flex-direction: column;
gap: 10px; gap: 10px;
width: min(100%, 40rem); width: min(100%, 40rem);
max-height: 30vh; min-height: 0;
overflow: hidden;
text-align: center; text-align: center;
} }
/* Each caption is its own bounded scroll region: the user transcript and the
reply overflow independently, scroll by touch inside the overlay without
moving the page behind (overscroll-behavior: contain), stay keyboard
scrollable via tabindex, and fade at the top edge so clipped history reads
as scrollable. Auto-follow of the streaming tail lives in atlas-voice.js
(data-follow, per region). */
.voice-conversation-caption-user,
.voice-conversation-caption-assistant {
overflow-y: auto;
touch-action: pan-y;
-webkit-overflow-scrolling: touch;
overscroll-behavior: contain;
scrollbar-width: thin;
-webkit-mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 100%);
mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 100%);
}
.voice-conversation-caption-user:focus-visible,
.voice-conversation-caption-assistant:focus-visible {
outline: 2px solid rgb(var(--voice-accent));
outline-offset: 2px;
}
.voice-conversation-caption-user { .voice-conversation-caption-user {
margin: 0; margin: 0;
max-height: 22vh;
font-size: 14px; font-size: 14px;
line-height: 1.5; line-height: 1.5;
color: rgba(178, 196, 214, 0.88); color: rgba(178, 196, 214, 0.88);
@ -478,6 +501,7 @@
.voice-conversation-caption-assistant { .voice-conversation-caption-assistant {
margin: 0; margin: 0;
max-height: 38vh;
font-size: 16px; font-size: 16px;
line-height: 1.6; line-height: 1.6;
color: rgba(240, 247, 255, 0.95); color: rgba(240, 247, 255, 0.95);
@ -550,7 +574,8 @@
@media (max-width: 640px) { @media (max-width: 640px) {
.voice-conversation { gap: 18px; } .voice-conversation { gap: 18px; }
.voice-conversation-captions { max-height: 34vh; } .voice-conversation-caption-user { max-height: 24vh; }
.voice-conversation-caption-assistant { max-height: 34vh; }
} }
/* Reduced motion: static orb states the accent color and the state caption /* Reduced motion: static orb states the accent color and the state caption

View File

@ -112,6 +112,11 @@
// bound to that turn's generation token and consumed exactly once. // bound to that turn's generation token and consumed exactly once.
let sttLanguage=''; let sttLanguage='';
let sttLanguageToken=-1; let sttLanguageToken=-1;
// Sticky per-hands-free-session language: set by the private Whisper
// detection of the user's own speech (or a strong reply-text signal on a
// detection-less turn), consumed as the NEXT streaming STT session's bias
// and as the thinking-cue locale. Cleared on activate/deactivate.
let sessionLanguage='';
const TTS_LANGUAGES=['en','ru','es']; const TTS_LANGUAGES=['en','ru','es'];
const originalAutoRead=window.autoReadLastAssistant; const originalAutoRead=window.autoReadLastAssistant;
const originalApplyPreference=window._applyVoiceModePref; const originalApplyPreference=window._applyVoiceModePref;
@ -138,6 +143,37 @@
return language; return language;
} }
const SPANISH_ORTHOGRAPHY=/[áéíóúñü¡¿]/i;
const SPANISH_STOPWORDS=/\b(?:el|la|los|las|un|una|es|está|qué|para|por|con|pero|como|más|sí|gracias|hola|puedo|también|muy|este|esta|todo|bien)\b/g;
function strongReplyLanguage(text){
// Script-level certainty only: Cyrillic text is Russian; Spanish
// orthography (accents, ñ, inverted punctuation) is Spanish. Plain-ASCII
// text yields no signal, so an English reply never flips a trusted
// STT-detected voice.
const sample=String(text||'').slice(0,400);
if(/[Ѐ-ӿ]/.test(sample)) return 'ru';
if(SPANISH_ORTHOGRAPHY.test(sample)) return 'es';
return '';
}
function detectReplyLanguage(text){
// Lightweight reply-language heuristic for turns without a trusted STT
// detection: script evidence first, then Spanish stopword density (an
// accent-free Spanish sentence still routes to the Spanish voice).
// Returns '' for English/unknown, which the private TTS service resolves
// to its own English default voice.
const strong=strongReplyLanguage(text);
if(strong) return strong;
const sample=String(text||'').slice(0,400).toLowerCase();
const words=sample.split(/\s+/).filter(Boolean);
if(words.length>=4){
const matches=(sample.match(SPANISH_STOPWORDS)||[]).length;
if(matches>=2&&matches/words.length>=0.12) return 'es';
}
return '';
}
function toast(message){ function toast(message){
if(typeof window.showToast==='function') window.showToast(message,3000); if(typeof window.showToast==='function') window.showToast(message,3000);
} }
@ -194,12 +230,32 @@
conversation.stateEl.textContent=customLabel||STATE_LABELS[next]||''; conversation.stateEl.textContent=customLabel||STATE_LABELS[next]||'';
} }
function updateCaptionRegion(element,text,limit){
// Captions are bounded scrollable regions: keep following the streaming
// tail unless the user scrolled up inside this region (data-follow='0',
// maintained by the scroll listener installed at overlay build time).
const value=String(text||'').slice(-limit);
element.textContent=value;
if(!value&&element.dataset) element.dataset.follow='1';
if((!element.dataset||element.dataset.follow!=='0')&&typeof element.scrollHeight==='number'){
try{element.scrollTop=element.scrollHeight;}catch(_){ }
}
}
function attachCaptionScroll(element){
if(!element.addEventListener) return;
element.addEventListener('scroll',function(){
const gap=(element.scrollHeight||0)-(element.scrollTop||0)-(element.clientHeight||0);
if(element.dataset) element.dataset.follow=gap<=24?'1':'0';
});
}
function setConversationUserCaption(text){ function setConversationUserCaption(text){
if(conversation) conversation.userCaption.textContent=String(text||'').slice(-600); if(conversation) updateCaptionRegion(conversation.userCaption,text,4000);
} }
function setConversationAssistantCaption(text){ function setConversationAssistantCaption(text){
if(conversation) conversation.assistantCaption.textContent=String(text||'').slice(-900); if(conversation) updateCaptionRegion(conversation.assistantCaption,text,9000);
} }
function setConversationPlaying(playing){ function setConversationPlaying(playing){
@ -261,8 +317,13 @@
orb.appendChild(conversationNode('span','voice-conversation-orb-ring')); orb.appendChild(conversationNode('span','voice-conversation-orb-ring'));
const stateEl=conversationNode('div','voice-conversation-state',{'aria-hidden':'true'}); const stateEl=conversationNode('div','voice-conversation-state',{'aria-hidden':'true'});
const captions=conversationNode('div','voice-conversation-captions',{'aria-live':'polite'}); const captions=conversationNode('div','voice-conversation-captions',{'aria-live':'polite'});
const userCaption=conversationNode('p','voice-conversation-caption-user'); // Each caption is its own bounded, independently scrollable region
const assistantCaption=conversationNode('p','voice-conversation-caption-assistant'); // (keyboard focusable, touch scrollable); streaming keeps following the
// tail until the user scrolls up inside that region.
const userCaption=conversationNode('p','voice-conversation-caption-user',{tabindex:'0'});
const assistantCaption=conversationNode('p','voice-conversation-caption-assistant',{tabindex:'0'});
attachCaptionScroll(userCaption);
attachCaptionScroll(assistantCaption);
captions.appendChild(userCaption); captions.appendChild(userCaption);
captions.appendChild(assistantCaption); captions.appendChild(assistantCaption);
const controls=conversationNode('div','voice-conversation-controls'); const controls=conversationNode('div','voice-conversation-controls');
@ -328,6 +389,7 @@
function cancelSpeechTurn(){ function cancelSpeechTurn(){
if(!speechTurn) return; if(!speechTurn) return;
cancelSpeakingIdleFallback(speechTurn);
speechTurn.cancelled=true; speechTurn.cancelled=true;
speechTurn.final=true; speechTurn.final=true;
while(speechTurn.waiters.length) speechTurn.waiters.shift()(null); while(speechTurn.waiters.length) speechTurn.waiters.shift()(null);
@ -453,6 +515,18 @@
if(ducked) indicator.classList.add('is-ducked'); else indicator.classList.remove('is-ducked'); if(ducked) indicator.classList.add('is-ducked'); else indicator.classList.remove('is-ducked');
} }
function playbackAudible(){
// True only while audio is actually sounding. A streaming PCM session
// keeps its worklet node alive across sentence chunks and between the
// speech segments of one turn; the buffered frames the worklet reports
// decide whether it is audible right now.
if(currentAudio) return true;
if(thinkingCue&&thinkingCue.node) return true;
const session=playbackSession;
if(session&&session.node) return session.bufferedFrames>0&&!session.playbackEnded;
return indicator.classList.contains('is-playing');
}
function disposeBargeResources(monitor,keepCapture){ function disposeBargeResources(monitor,keepCapture){
if(!monitor) return; if(!monitor) return;
monitor.cancelled=true; monitor.cancelled=true;
@ -570,6 +644,7 @@
clearErrorTimer(); clearErrorTimer();
removeConversationOverlay(); removeConversationOverlay();
clearSttLanguage(); clearSttLanguage();
sessionLanguage='';
suppressAutoRead=false; suppressAutoRead=false;
pendingStitch=null; pendingStitch=null;
lastSentTranscript=null; lastSentTranscript=null;
@ -595,25 +670,117 @@
},delay||500); },delay||500);
} }
function resyncCapture(token,statusLabel){
// Full capture-turn resync: cancel any streaming STT session, drop the
// lookback/pending buffers and start a fresh capture turn on the hot
// microphone. Used after an errored turn and by barge-in edge cases
// where the previous stream state is no longer trustworthy.
if(!active||token!==generation) return;
stopCapture();
startListening(token);
setState('listening',statusLabel);
}
function handleAssistantResponseError(token){
// An error/system envelope (cancellation notice, provider failure) is a
// transcript artifact, not a reply: never feed it to TTS or the reply
// caption, show a brief non-spoken state instead, and resynchronize
// capture so the next utterance starts a clean streaming turn.
thinkingSession=null;
thinkingTurnId='';
clearSttLanguage();
stopResponseObserver();
cancelThinkingCues();
cancelSpeechTurn();
stopPlayback();
pendingStitch=null;
lastSentTranscript=null;
clearBargeCancellation();
setConversationAssistantCaption('');
resyncCapture(token,'Something went wrong — listening');
}
function assistantRows(){ function assistantRows(){
return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]'); return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
} }
function readAssistantRow(row){ function assistantTurnOf(row){
if(!row) return ''; // Resolve a matched node to its whole turn container so multi-segment
if(row.dataset&&typeof row.dataset.rawText==='string') return row.dataset.rawText; // turns (interim messages around tool calls) are read as one response.
return typeof row.textContent==='string'?row.textContent:''; if(row&&typeof row.closest==='function'){
const turn=row.closest('.msg-row[data-role="assistant"]');
if(turn) return turn;
}
return row;
}
function segmentIsHidden(segment){
if(!segment) return true;
if(segment.hidden===true) return true;
return !!(typeof segment.getAttribute==='function'&&segment.getAttribute('aria-hidden')==='true');
}
function segmentIsError(segment){
// Error/system envelopes: the renderer stamps data-error="1" on segments
// matching its error patterns, and provider errors and cancellation
// notices carry a .provider-error-details block inside the body.
if(!segment) return false;
if(segment.dataset&&segment.dataset.error==='1') return true;
return !!(typeof segment.querySelector==='function'&&segment.querySelector('.provider-error-details'));
}
function readSegmentBody(segment){
if(!segment) return '';
if(segment.dataset&&typeof segment.dataset.rawText==='string') return segment.dataset.rawText;
// Scrape only message BODY text — never the avatar letter, author name,
// "Processed Ns" worklog chips or footer controls that share the row.
if(typeof segment.querySelectorAll==='function'){
const bodies=segment.querySelectorAll('.msg-body');
if(bodies&&bodies.length){
return Array.prototype.map.call(bodies,function(body){return body.textContent||'';}).join('\n');
}
}
return typeof segment.textContent==='string'?segment.textContent:'';
}
function readAssistantTurn(turn){
// {text, error}: body-only text of every visible answer segment in the
// turn, plus whether any segment is an error/system envelope.
if(!turn) return {text:'',error:false};
let error=false;
const parts=[];
const segments=(typeof turn.querySelectorAll==='function')?turn.querySelectorAll('.assistant-segment'):null;
if(segments&&segments.length){
Array.prototype.forEach.call(segments,function(segment){
if(segmentIsHidden(segment)) return;
if(segmentIsError(segment)){error=true;return;}
const value=readSegmentBody(segment);
if(value&&value.trim()) parts.push(value.trim());
});
}else if(segmentIsError(turn)){
error=true;
}else{
const value=readSegmentBody(turn);
if(value&&value.trim()) parts.push(value.trim());
}
return {text:cleanForSpeech(parts.join('\n\n')),error:error};
} }
function rememberAssistantBaseline(){ function rememberAssistantBaseline(){
const rows=assistantRows(); const rows=assistantRows();
const row=rows.length?rows[rows.length-1]:null; const turn=rows.length?assistantTurnOf(rows[rows.length-1]):null;
assistantBaseline={row:row,text:readAssistantRow(row),count:rows.length}; assistantBaseline={row:turn,text:readAssistantTurn(turn).text,count:rows.length};
} }
async function settleBargeCancellation(token){ async function settleBargeCancellation(token){
const cancellation=bargeCancelPromise; const cancellation=bargeCancelPromise;
if(!cancellation) return true; if(!cancellation) return true;
if(!cancellation.streamId){
// Nothing was actually in flight to cancel: a stale busy flag left by
// an errored turn must never hold the send in a dead settle wait.
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
return true;
}
try{await cancellation.promise;}catch(_){ } try{await cancellation.promise;}catch(_){ }
const deadline=Date.now()+10000; const deadline=Date.now()+10000;
while(active&&token===generation&&Date.now()<deadline){ while(active&&token===generation&&Date.now()<deadline){
@ -642,7 +809,22 @@
const sent=lastSentTranscript; const sent=lastSentTranscript;
lastSentTranscript=null; lastSentTranscript=null;
if(!sent||!sent.text) return; if(!sent||!sent.text) return;
pendingStitch={text:sent.text,at:Date.now()}; // The playback queue knows exactly where the user stopped listening: the
// chunk being spoken (or the last fully spoken one) becomes the cut
// marker appended to the stitched follow-up message.
const turn=speechTurn;
const cut=turn?String(turn.speakingChunk||turn.lastSpokenChunk||''):'';
pendingStitch={text:sent.text,at:Date.now(),cut:cut};
}
function voiceCutMarker(cut){
// Exactly one compact machine-readable line, appended to a stitched
// interruption, telling the model where its spoken reply was cut off so
// it can resume naturally instead of re-answering from the top.
const tail=String(cut||'').replace(/\s+/g,' ').replace(/"/g,"'").trim();
if(!tail) return '';
const bounded=tail.length>120?tail.slice(tail.length-120):tail;
return '\n[voice interruption: you were cut off after "'+bounded+'"]';
} }
async function sendTranscript(transcript,token,language,turnId){ async function sendTranscript(transcript,token,language,turnId){
@ -662,7 +844,7 @@
// restated as one stitched message, so the model answers the full thought. // restated as one stitched message, so the model answers the full thought.
const stitch=(pendingStitch&&pendingStitch.text&&(Date.now()-pendingStitch.at)<=STITCH_WINDOW_MS)?pendingStitch:null; const stitch=(pendingStitch&&pendingStitch.text&&(Date.now()-pendingStitch.at)<=STITCH_WINDOW_MS)?pendingStitch:null;
pendingStitch=null; pendingStitch=null;
composer.value=stitch?stitch.text+' '+text:text; composer.value=stitch?stitch.text+' '+text+voiceCutMarker(stitch.cut):text;
if(typeof window.autoResize==='function') window.autoResize(); if(typeof window.autoResize==='function') window.autoResize();
setConversationUserCaption(composer.value); setConversationUserCaption(composer.value);
setConversationAssistantCaption(''); setConversationAssistantCaption('');
@ -675,19 +857,34 @@
if(!cancellationSettled){ if(!cancellationSettled){
pendingStitch=stitch; pendingStitch=stitch;
toast('The previous response did not stop. Please repeat your interruption.'); toast('The previous response did not stop. Please repeat your interruption.');
restartSoon(token,250); // The stream state is now suspect: rebuild the capture turn instead of
// resuming a session whose server-side epoch may be stale.
resyncCapture(token);
return; return;
} }
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null; thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
thinkingTurnId=turnId||captureTurnId||String(token)+'-'+String(++turnSequence); thinkingTurnId=turnId||captureTurnId||String(token)+'-'+String(++turnSequence);
rememberAssistantBaseline(); rememberAssistantBaseline();
rememberSttLanguage(language,token); rememberSttLanguage(language,token);
// The user's detected speech language is the sticky hands-free session
// language: it biases the next streaming STT session and localizes the
// thinking cues until the user audibly switches again.
if(language){
const switched=language!==sessionLanguage;
sessionLanguage=language;
// Continuous capture already opened the next session (with the
// previous bias) before this turn's detection arrived: restart it with
// the switched language, but only while it has heard nothing yet.
if(switched&&captureActive&&streamingStt&&typeof streamingStt.latestPartial==='function'&&!streamingStt.latestPartial()){
startListening(generation,{preserveDisplay:true});
}
}
if(typeof window.send==='function'){ if(typeof window.send==='function'){
lastSentTranscript={text:composer.value,token:token}; lastSentTranscript={text:composer.value,token:token};
window.send(); window.send();
suppressAutoRead=false; suppressAutoRead=false;
startResponseObserver(token); startResponseObserver(token);
scheduleThinkingCues(token,language,thinkingTurnId); scheduleThinkingCues(token,language||sessionLanguage,thinkingTurnId);
} }
} }
@ -798,6 +995,7 @@
let workletReady=false; let workletReady=false;
let partialRevision=-1; let partialRevision=-1;
let lastPreflightText=''; let lastPreflightText='';
let lastPartialText='';
let queue=[]; let queue=[];
let queuedBytes=0; let queuedBytes=0;
let flushTimer=null; let flushTimer=null;
@ -862,7 +1060,9 @@
return null; return null;
} }
socket.onopen=function(){ socket.onopen=function(){
sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:'auto'}); // Bias recognition toward the session's sticky language the moment the
// user has audibly switched; 'auto' remains the cold-start default.
sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:sessionLanguage||'auto'});
flush(); flush();
}; };
socket.onmessage=function(event){ socket.onmessage=function(event){
@ -877,6 +1077,10 @@
const stable=String(payload.stable_transcript||'').trim(); const stable=String(payload.stable_transcript||'').trim();
const provisional=String(payload.transcript||'').trim(); const provisional=String(payload.transcript||'').trim();
const visible=stable||provisional; const visible=stable||provisional;
// The newest rolling partial also feeds the dynamic endpoint below,
// so it is tracked regardless of the display-only gating that keeps
// the label and captions quiet outside the listening state.
if(visible) lastPartialText=visible;
if(visible&&active&&captureTurnId===turnId&&state==='listening'){ if(visible&&active&&captureTurnId===turnId&&state==='listening'){
const preview=visible.length>72?visible.slice(0,69)+'…':visible; const preview=visible.length>72?visible.slice(0,69)+'…':visible;
label.textContent='Listening · '+preview+(stable?'':' · provisional'); label.textContent='Listening · '+preview+(stable?'':' · provisional');
@ -904,6 +1108,7 @@
return { return {
finalPromise:finalPromise, finalPromise:finalPromise,
latestPartial:function(){return lastPartialText;},
setWorkletReady:function(value){workletReady=value;}, setWorkletReady:function(value){workletReady=value;},
push:function(samples){ push:function(samples){
if(cancelled||committed||!workletReady) return; if(cancelled||committed||!workletReady) return;
@ -1149,7 +1354,7 @@
for(let index=0;index<samples.length;index+=1){const value=(samples[index]-128)/128;energy+=value*value;} for(let index=0;index<samples.length;index+=1){const value=(samples[index]-128)/128;energy+=value*value;}
const rms=Math.sqrt(energy/samples.length); const rms=Math.sqrt(energy/samples.length);
const now=Date.now(); const now=Date.now();
const playbackActive=indicator.classList.contains('is-playing')||!!currentAudio||!!(playbackSession&&playbackSession.node)||!!(thinkingCue&&thinkingCue.node); const playbackActive=playbackAudible();
if(playbackActive&&!playbackWasActive){ if(playbackActive&&!playbackWasActive){
// AEC needs a brief convergence window when local speech starts. PCM // AEC needs a brief convergence window when local speech starts. PCM
// lookback preserves a real interruption spoken during this guard. // lookback preserves a real interruption spoken during this guard.
@ -1387,7 +1592,7 @@
// AEC nothing counts as speech at all, and with AEC a post-playback // AEC nothing counts as speech at all, and with AEC a post-playback
// arming delay plus the sustained-frame requirement below mirror the // arming delay plus the sustained-frame requirement below mirror the
// conservative energy monitor this path supersedes. // conservative energy monitor this path supersedes.
const playbackLive=indicator.classList.contains('is-playing')||!!currentAudio||!!(playbackSession&&playbackSession.node)||!!(thinkingCue&&thinkingCue.node); const playbackLive=playbackAudible();
if(playbackLive&&!playbackWasLive){ if(playbackLive&&!playbackWasLive){
bargeArmAt=now+(state==='thinking'?150:450); bargeArmAt=now+(state==='thinking'?150:450);
}else if(!playbackLive&&playbackWasLive){ }else if(!playbackLive&&playbackWasLive){
@ -1433,8 +1638,15 @@
speculative=true; speculative=true;
} }
// A young utterance holds its endpoint longer: pausing to think right // A young utterance holds its endpoint longer: pausing to think right
// after the first word must not send a one-word fragment. // after the first word must not send a one-word fragment. When the
const endpointSilenceMs=speechMs<VAD_COMMITTED_SPEECH_MS?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs; // streaming partial already reads as a plausibly complete utterance
// (>=3 words, or terminal punctuation), endpoint at the base window
// instead so short commands are not delayed by the clipping guard —
// the long hold remains only for 1-2 word partials.
const partialText=streamingStt&&streamingStt.latestPartial?streamingStt.latestPartial():'';
const partialWords=partialText?partialText.split(/\s+/).filter(Boolean).length:0;
const partialComplete=partialWords>=3||(partialWords>0&&/[.!?…]["')\]}]*$/.test(partialText));
const endpointSilenceMs=(speechMs<VAD_COMMITTED_SPEECH_MS&&!partialComplete)?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs;
const finished=heardSpeech&&(now-lastSpeech)>=endpointSilenceMs; const finished=heardSpeech&&(now-lastSpeech)>=endpointSilenceMs;
const timedOut=now-started>=90000; const timedOut=now-started>=90000;
const idle=(!heardSpeech)&&(now-started)>=20000; const idle=(!heardSpeech)&&(now-started)>=20000;
@ -1775,7 +1987,7 @@
streamingCapability.tts=null; streamingCapability.tts=null;
} }
} }
return {kind:'blob',blob:await fetchSpeech(chunk,language,turnId,token)}; return {kind:'blob',chunk:chunk,blob:await fetchSpeech(chunk,language,turnId,token)};
} }
async function ensurePcmPlayback(asset,token){ async function ensurePcmPlayback(asset,token){
@ -1923,6 +2135,25 @@
while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null); while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null);
} }
function scheduleSpeakingIdleFallback(turn,session){
// Interim-message turns speak in cycles (speak → tools → speak): once no
// further chunk is queued and the turn is not final, fall back to
// Thinking after the buffered audio has played out. The observer flips
// the state back to Speaking when the next segment yields a chunk.
cancelSpeakingIdleFallback(turn);
const bufferedMs=session&&session.sampleRate?Math.ceil(((session.bufferedFrames||0)/session.sampleRate)*1000):0;
turn.idleTimer=window.setTimeout(function(){
turn.idleTimer=null;
if(!active||turn.token!==generation||turn.cancelled||speechTurn!==turn) return;
if(turn.queue.length||turn.final) return;
if(state==='speaking') setState('thinking');
},bufferedMs+250);
}
function cancelSpeakingIdleFallback(turn){
if(turn&&turn.idleTimer){window.clearTimeout(turn.idleTimer);turn.idleTimer=null;}
}
async function runSpeechQueue(turn){ async function runSpeechQueue(turn){
if(turn.running) return; if(turn.running) return;
turn.running=true; turn.running=true;
@ -1943,9 +2174,15 @@
// Barge-in can abort both the playing request and its one-ahead request. // Barge-in can abort both the playing request and its one-ahead request.
// Observe the latter even when the cancelled current turn returns first. // Observe the latter even when the cancelled current turn returns first.
nextPrepared.catch(function(){ }); nextPrepared.catch(function(){ });
turn.speakingChunk=asset.chunk||'';
await playPrepared(asset,turn.token); await playPrepared(asset,turn.token);
turn.lastSpokenChunk=asset.chunk||turn.lastSpokenChunk;
turn.speakingChunk='';
if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return; if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return;
if(!turn.queue.length&&!turn.final) scheduleSpeakingIdleFallback(turn,session);
current=await nextPrepared; current=await nextPrepared;
cancelSpeakingIdleFallback(turn);
if(current) setState('speaking');
} }
await drainPcmPlayback(session,turn.token); await drainPcmPlayback(session,turn.token);
if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300); if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300);
@ -1962,23 +2199,34 @@
} }
} }
function currentAssistantText(){ function collectAssistantResponse(){
const rows=assistantRows(); const rows=assistantRows();
if(!rows.length) return ''; if(!rows.length) return {text:'',error:false};
const row=rows[rows.length-1]; const turn=assistantTurnOf(rows[rows.length-1]);
const text=readAssistantRow(row); const response=readAssistantTurn(turn);
if(assistantBaseline&&row===assistantBaseline.row&&text===assistantBaseline.text) return ''; if(assistantBaseline){
if(assistantBaseline&&rows.length<assistantBaseline.count) return ''; if(rows.length<assistantBaseline.count) return {text:'',error:false};
return cleanForSpeech(text); if(turn===assistantBaseline.row&&response.text===assistantBaseline.text) return {text:'',error:response.error};
}
return response;
}
function currentAssistantText(){
return collectAssistantResponse().text;
} }
function ensureSpeechTurn(token){ function ensureSpeechTurn(token){
if(speechTurn) return speechTurn; if(speechTurn) return speechTurn;
const sttDetected=takeSttLanguage(token);
speechTurn={ speechTurn={
token:token, token:token,
turnId:thinkingTurnId||String(token)+'-'+String(++turnSequence), turnId:thinkingTurnId||String(token)+'-'+String(++turnSequence),
language:takeSttLanguage(token), language:sttDetected,
sttLanguage:sttDetected,
sourceText:'', sourceText:'',
speakingChunk:'',
lastSpokenChunk:'',
idleTimer:null,
consumed:0, consumed:0,
first:true, first:true,
queue:[], queue:[],
@ -2009,7 +2257,12 @@
restartSoon(token,250); restartSoon(token,250);
return; return;
} }
const text=currentAssistantText(); const response=collectAssistantResponse();
if(response.error){
handleAssistantResponseError(token);
return;
}
const text=response.text;
if(!text){ if(!text){
if(isFinal){thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();cancelThinkingCues();restartSoon(token,250);} if(isFinal){thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();cancelThinkingCues();restartSoon(token,250);}
return; return;
@ -2020,6 +2273,18 @@
// of the answer queue. // of the answer queue.
cancelThinkingCues(); cancelThinkingCues();
const turn=ensureSpeechTurn(token); const turn=ensureSpeechTurn(token);
if(turn.first){
// Reply-voice routing, finalized when the first chunk is cut: the
// turn's STT-detected language wins, but script evidence in the reply
// text corrects a wrong or missing detection (a Spanish reply must
// never be spoken by the English Amy voice), and a detection-less turn
// falls back to the reply-text heuristic. A correction on a
// detection-less turn also re-biases the sticky session language so
// the next streaming STT turn recognizes the switch promptly.
const resolved=strongReplyLanguage(text)||turn.sttLanguage||detectReplyLanguage(text);
if(resolved) turn.language=resolved;
if(!turn.sttLanguage&&resolved) sessionLanguage=resolved;
}
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){ if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
// Renderers can revise the still-unspoken tail. Already-spoken text is // Renderers can revise the still-unspoken tail. Already-spoken text is
// immutable. On the completion callback, advance from the old spoken // immutable. On the completion callback, advance from the old spoken
@ -2099,6 +2364,7 @@
active=true; active=true;
clearErrorTimer(); clearErrorTimer();
clearSttLanguage(); clearSttLanguage();
sessionLanguage='';
modeBtn.classList.add('active'); modeBtn.classList.add('active');
toast('Hands-free private voice mode on'); toast('Hands-free private voice mode on');
openConversationOverlay(); openConversationOverlay();

View File

@ -71,11 +71,28 @@
} }
.hux-workspace-toggle:focus-visible, .hux-workspace-toggle:focus-visible,
.hux-workspace-toggle-rail:focus-visible,
.hux-workspace-drawer button:focus-visible { .hux-workspace-drawer button:focus-visible {
outline: 2px solid #5ee7d7; outline: 2px solid #5ee7d7;
outline-offset: 2px; outline-offset: 2px;
} }
/* Rail-mounted Workspace toggle: the upstream .rail-btn/.has-tooltip rules
own its look and tooltip; only the open-drawer state is marked here. */
.hux-workspace-toggle-rail[aria-expanded="true"] {
background: var(--accent-bg, rgb(94 231 215 / 18%));
color: var(--accent-text, var(--text, #e7eef6));
}
/* Where the rail is visible (upstream shows it from 641px up) the floating
fallback toggle disappears; below that the rail itself is hidden and the
floating button remains the only entry point, so no width loses access. */
@media (min-width: 641px) {
.hux-workspace-toggle--railed {
display: none;
}
}
@media (max-width: 640px) { @media (max-width: 640px) {
.hux-workspace-drawer { .hux-workspace-drawer {
width: 100%; width: 100%;

View File

@ -232,9 +232,53 @@
return value; return value;
} }
function railHost(doc) {
// The app's left icon rail (chat, tasks, telegram, …). Optional: bundles
// without it (and headless harnesses) keep the floating toggle.
if (typeof doc.querySelector !== 'function') return null;
const rail = doc.querySelector('nav.rail');
return rail && typeof rail.insertBefore === 'function' ? rail : null;
}
function railIcon(doc) {
try {
if (typeof doc.createElementNS !== 'function') return null;
const svgNS = 'http://www.w3.org/2000/svg';
const svg = doc.createElementNS(svgNS, 'svg');
[['width', '20'], ['height', '20'], ['viewBox', '0 0 24 24'], ['fill', 'none'],
['stroke', 'currentColor'], ['stroke-width', '1.5'], ['stroke-linecap', 'round'],
['stroke-linejoin', 'round'], ['aria-hidden', 'true'],
].forEach(([name, value]) => svg.setAttribute(name, value));
const path = doc.createElementNS(svgNS, 'path');
path.setAttribute('d', 'M3 5a2 2 0 0 1 2-2h6v18H5a2 2 0 0 1-2-2V5Zm10-2h6a2 2 0 0 1 2 2v5h-8V3Zm0 11h8v5a2 2 0 0 1-2 2h-6v-7Z');
svg.appendChild(path);
return svg;
} catch (_) {
return null;
}
}
function createChrome(doc) { function createChrome(doc) {
const open = node(doc, 'button', {'class': 'hux-workspace-toggle', type: 'button', const rail = railHost(doc);
'aria-controls': 'huxWorkspaceDrawer', 'aria-expanded': 'false'}, 'Workspace'); const open = node(doc, 'button', {'class': rail ?
'hux-workspace-toggle hux-workspace-toggle--railed' : 'hux-workspace-toggle',
type: 'button', 'aria-controls': 'huxWorkspaceDrawer', 'aria-expanded': 'false'}, 'Workspace');
let railToggle = null;
if (rail) {
// Rail placement: an icon-only item beside the app's own tabs, styled
// by the upstream .rail-btn/.has-tooltip rules. The floating button
// stays in the DOM as the below-rail-breakpoint fallback and is hidden
// by bootstrap.css wherever the rail is visible, so nothing floats
// over the chat. Drawer behavior is unchanged.
railToggle = node(doc, 'button', {'class': 'rail-btn has-tooltip hux-workspace-toggle-rail',
type: 'button', 'aria-controls': 'huxWorkspaceDrawer', 'aria-expanded': 'false',
'data-tooltip': 'Workspace', 'aria-label': 'Workspace'});
const icon = railIcon(doc);
if (icon) railToggle.appendChild(icon);
else railToggle.textContent = '⧉';
const spacer = typeof rail.querySelector === 'function' ? rail.querySelector('.rail-spacer') : null;
rail.insertBefore(railToggle, spacer || null);
}
const drawer = node(doc, 'aside', {id: 'huxWorkspaceDrawer', 'class': 'hux-workspace-drawer', const drawer = node(doc, 'aside', {id: 'huxWorkspaceDrawer', 'class': 'hux-workspace-drawer',
'aria-labelledby': 'huxWorkspaceTitle'}); 'aria-labelledby': 'huxWorkspaceTitle'});
const header = node(doc, 'header', {'class': 'hux-workspace-drawer__header'}); const header = node(doc, 'header', {'class': 'hux-workspace-drawer__header'});
@ -246,16 +290,19 @@
doc.body.appendChild(open); doc.body.appendChild(drawer); doc.body.appendChild(open); doc.body.appendChild(drawer);
function setOpen(value) { function setOpen(value) {
drawer.hidden = !value; open.setAttribute('aria-expanded', value ? 'true' : 'false'); drawer.hidden = !value; open.setAttribute('aria-expanded', value ? 'true' : 'false');
if (railToggle) railToggle.setAttribute('aria-expanded', value ? 'true' : 'false');
if (value && typeof close.focus === 'function') close.focus(); if (value && typeof close.focus === 'function') close.focus();
} }
const onOpen = () => setOpen(true); const onOpen = () => setOpen(true);
const onClose = () => setOpen(false); const onClose = () => setOpen(false);
const onKey = (event) => { if (event.key === 'Escape' && !drawer.hidden) setOpen(false); }; const onKey = (event) => { if (event.key === 'Escape' && !drawer.hidden) setOpen(false); };
open.addEventListener('click', onOpen); close.addEventListener('click', onClose); open.addEventListener('click', onOpen); close.addEventListener('click', onClose);
if (railToggle) railToggle.addEventListener('click', onOpen);
doc.addEventListener('keydown', onKey); doc.addEventListener('keydown', onKey);
return Object.freeze({content, drawer, open, setOpen, destroy() { return Object.freeze({content, drawer, open, railToggle, setOpen, destroy() {
doc.removeEventListener('keydown', onKey); doc.removeEventListener('keydown', onKey);
if (open.parentNode) open.parentNode.removeChild(open); if (open.parentNode) open.parentNode.removeChild(open);
if (railToggle && railToggle.parentNode) railToggle.parentNode.removeChild(railToggle);
if (drawer.parentNode) drawer.parentNode.removeChild(drawer); if (drawer.parentNode) drawer.parentNode.removeChild(drawer);
}}); }});
} }

View File

@ -0,0 +1,80 @@
"""Sliced upstream fixture: the exact api/profiles.py fragments the Atlas
voice patcher grafts into (session-continuity 409 resilience). Mirrors the
pinned Hermes WebUI deployment byte-for-byte for the anchored regions; see
dockerfiles/hermes-webui-atlas-patch.py."""
import logging
import threading
logger = logging.getLogger(__name__)
_root_profile_name_cache = {'default'}
_root_profile_name_cache_lock = threading.Lock()
_root_profile_name_cache_loaded = False
def list_profiles_api():
"""Fixture stand-in for the hermes_cli-backed profile listing."""
return []
def _is_root_profile(name: str) -> bool:
"""True if *name* resolves to the Hermes Agent root profile (~/.hermes).
Matches the legacy 'default' alias plus any name where list_profiles_api()
reports is_default=True. Memoized; call _invalidate_root_profile_cache()
after mutating profile metadata.
"""
global _root_profile_name_cache_loaded
if not name:
return False
if name == 'default':
return True
with _root_profile_name_cache_lock:
if _root_profile_name_cache_loaded:
return name in _root_profile_name_cache
# Cache miss — populate from list_profiles_api(). Done outside the lock to
# avoid holding it across a hermes_cli subprocess call.
try:
infos = list_profiles_api()
except Exception:
logger.debug("Failed to list profiles for root-profile lookup", exc_info=True)
return False
with _root_profile_name_cache_lock:
_root_profile_name_cache.clear()
_root_profile_name_cache.add('default')
for p in infos:
try:
if p.get('is_default') and p.get('name'):
_root_profile_name_cache.add(p['name'])
except (AttributeError, TypeError):
continue
_root_profile_name_cache_loaded = True
return name in _root_profile_name_cache
def _profiles_match(row_profile, active_profile) -> bool:
"""Return True if a session/project row's profile matches the active profile.
Treats both the literal alias 'default' and any renamed-root display name
(per _is_root_profile) as equivalent, so legacy rows tagged 'default'
still surface when the user has renamed the root profile to e.g. 'kinni',
and vice versa.
A row with no profile (`None` or empty string) is treated as belonging to
the root profile that's the convention used by the legacy backfill at
api/models.py::all_sessions, and matches the default seen in
`static/sessions.js` (`S.activeProfile||'default'`).
Originally lived in api/routes.py; relocated here so both routes.py and
out-of-process consumers (mcp_server.py) can import the canonical helper
instead of duplicating the body. See #1614 for the visibility model.
"""
row = row_profile or 'default'
active = active_profile or 'default'
if row == active:
return True
# Cross-alias the renamed root.
if _is_root_profile(row) and _is_root_profile(active):
return True
return False

View File

@ -233,6 +233,7 @@ function makeHarness(options = {}) {
const sends = []; const sends = [];
const toasts = []; const toasts = [];
const transcribeUploads = []; const transcribeUploads = [];
const ttsCalls = [];
const servers = []; const servers = [];
const captureNodes = []; const captureNodes = [];
const analysers = []; const analysers = [];
@ -358,7 +359,7 @@ function makeHarness(options = {}) {
if (this.readyState !== 1) throw new Error('socket not open'); if (this.readyState !== 1) throw new Error('socket not open');
if (typeof data === 'string') { if (typeof data === 'string') {
const message = JSON.parse(data); const message = JSON.parse(data);
if (message.type === 'start') { this.turnId = message.turn_id; return; } if (message.type === 'start') { this.turnId = message.turn_id; this.startLanguage = message.language; return; }
if (message.type === 'speculate') { this.server.speculate(); return; } if (message.type === 'speculate') { this.server.speculate(); return; }
if (message.type === 'resume') { this.server.resume(); return; } if (message.type === 'resume') { this.server.resume(); return; }
if (message.type === 'cancel') { this.server.events.push('cancel'); return; } if (message.type === 'cancel') { this.server.events.push('cancel'); return; }
@ -374,6 +375,22 @@ function makeHarness(options = {}) {
return; return;
} }
this.server.append(new Int16Array(data)); this.server.append(new Int16Array(data));
// Mirror the server's rolling-partial stream: one stable partial per
// decoded-transcript change, so the client's dynamic endpoint sees the
// same signal the Jetson emits.
const partialText = this.server.transcript();
if (partialText && partialText !== this.lastPartialText) {
this.lastPartialText = partialText;
this.partialRevision = (this.partialRevision || 0) + 1;
this.deliver({
type: 'partial',
rolling: true,
turn_id: this.turnId,
revision: this.partialRevision,
transcript: partialText,
stable_transcript: partialText,
});
}
} }
close(code) { this.readyState = 3; this.closeCode = code; } close(code) { this.readyState = 3; this.closeCode = code; }
} }
@ -398,6 +415,7 @@ function makeHarness(options = {}) {
return { ok: true, status: 200, json: async () => ({ transcript: 'CONTAINER-FALLBACK', language: 'en' }) }; return { ok: true, status: 200, json: async () => ({ transcript: 'CONTAINER-FALLBACK', language: 'en' }) };
} }
if (url === '/api/tts') { if (url === '/api/tts') {
ttsCalls.push(JSON.parse((init && init.body) || '{}'));
return { ok: true, status: 200, blob: async () => ({ synthetic: true }), json: async () => ({}) }; return { ok: true, status: 200, blob: async () => ({ synthetic: true }), json: async () => ({}) };
} }
if (String(url).indexOf('api/chat/cancel') >= 0) { if (String(url).indexOf('api/chat/cancel') >= 0) {
@ -580,6 +598,7 @@ function makeHarness(options = {}) {
sends, sends,
toasts, toasts,
transcribeUploads, transcribeUploads,
ttsCalls,
servers, servers,
recorders, recorders,
clock, clock,
@ -589,6 +608,7 @@ function makeHarness(options = {}) {
flush, flush,
runDueTimeouts, runDueTimeouts,
setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; }, setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; },
setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; },
clearAssistantRows() { assistantRows = []; }, clearAssistantRows() { assistantRows = []; },
state() { return elements.voiceModeBar.dataset.voiceState || ''; }, state() { return elements.voiceModeBar.dataset.voiceState || ''; },
body() { return bodyElement; }, body() { return bodyElement; },
@ -773,6 +793,95 @@ scenarios.conversation_overlay_lifecycle = async () => {
}; };
}; };
// Dynamic endpointing: a >=3-word partial reads as a plausibly complete
// utterance and endpoints at the base silence window; 1-2 word partials keep
// the young-utterance hold so thinking pauses still never clip.
scenarios.three_word_partial_endpoints_at_base_silence = async () => {
const harness = makeHarness();
await harness.start();
await harness.silence(300);
await harness.speak('alpha', 300);
await harness.speak('bravo', 300);
await harness.speak('charlie', 400);
await harness.silence(1200); // past the base 1100ms endpoint, well under the 1800ms hold
await harness.tick(300);
return { sends: harness.sends, state: harness.state() };
};
scenarios.two_word_young_utterance_keeps_the_hold = async () => {
const harness = makeHarness();
await harness.start();
await harness.silence(300);
await harness.speak('alpha', 300);
await harness.speak('bravo', 300);
await harness.silence(1300); // longer than base, shorter than the hold
const sendsEarly = harness.sends.slice();
await harness.silence(700);
await harness.tick(300);
return { sendsEarly, sends: harness.sends };
};
// An errored turn (error/system envelope in the transcript) is never spoken,
// never captioned as a reply, and fully resynchronizes capture.
scenarios.errored_turn_is_not_spoken_and_capture_resyncs = async () => {
const harness = makeHarness();
await harness.start();
await harness.silence(300);
await harness.speak('alpha', 1300);
await harness.silence(1500);
await harness.tick(400); // dispatched; thinking
const serversBefore = harness.servers.length;
const ttsBefore = harness.ttsCalls.length;
harness.setAssistantError('**Task cancelled:** Task cancelled.');
await harness.tick(300); // the response observer sees the envelope
const stateAfterError = harness.state();
const labelAfterError = harness.elements.voiceModeLabel.textContent;
await harness.speak('bravo', 1300);
await harness.silence(1500);
await harness.tick(400);
return {
stateAfterError,
labelAfterError,
ttsDuringError: harness.ttsCalls.length - ttsBefore,
freshSessions: harness.servers.length - serversBefore,
sends: harness.sends,
};
};
// Voice barge-in appends a single-line cut marker naming the sentence that
// was playing, so the model knows where its reply was cut off.
scenarios.barge_cut_marker_records_spoken_tail = async () => {
const harness = makeHarness();
await harness.start();
await harness.silence(300);
await harness.speak('alpha', 1300);
await harness.silence(1500);
await harness.tick(400); // dispatched; thinking
harness.setAssistantReply('The first point is ready. The second point needs many more words before it becomes another speakable chunk.');
await harness.tick(300); // first sentence reaches TTS and starts playing
await harness.speak('bravo', 900); // talk over the reply
await harness.silence(1900);
await harness.tick(400);
return { sends: harness.sends, ttsTexts: harness.ttsCalls.map((request) => request.text) };
};
// The detected utterance language re-biases the following streaming STT
// session (sticky per hands-free session).
scenarios.sticky_language_biases_next_stt_session = async () => {
const harness = makeHarness();
await harness.start();
await harness.silence(300);
await harness.speak('alpha', 1300);
await harness.silence(1500);
await harness.tick(400);
harness.completeResponse();
await harness.tick(400);
await harness.speak('bravo', 1300);
await harness.silence(1500);
await harness.tick(400);
return { startLanguages: harness.servers.map((socket) => socket.startLanguage) };
};
(async () => { (async () => {
const output = {}; const output = {};
for (const name of Object.keys(scenarios)) { for (const name of Object.keys(scenarios)) {

View File

@ -520,6 +520,21 @@ scenarios.spoken_urls_are_elided_before_all_voice_routes = async () => {
return { results }; return { results };
}; };
scenarios.spanish_reply_without_detection_uses_reply_heuristic = async () => {
const harness = makeHarness();
await startVoiceMode(harness);
await runTurn(harness, { transcript: 'Que tal', reply: '¿Claro que sí! Todo está listo para continuar.' });
return { tts: harness.ttsRequests };
};
scenarios.reply_script_evidence_corrects_wrong_detection = async () => {
const harness = makeHarness();
await startVoiceMode(harness);
await runTurn(harness, { transcript: 'Answer in Spanish please.', language: 'en',
reply: 'Está bien: la razón es fácil de explicar.' });
return { tts: harness.ttsRequests };
};
scenarios.canonical_pcm_fallback_builds_a_valid_wav = async () => { scenarios.canonical_pcm_fallback_builds_a_valid_wav = async () => {
const harness = makeHarness(); const harness = makeHarness();
await harness.flush(); await harness.flush();

View File

@ -10,6 +10,8 @@ import shutil
import subprocess import subprocess
import sys import sys
import pytest
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js" VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js"
@ -135,6 +137,35 @@ def test_local_conversion_accepts_browser_fallback_containers(tmp_path: Path):
module = _patched_transcription_module(tmp_path) module = _patched_transcription_module(tmp_path)
formats = ((".ogg", "libopus"), (".mp4", "aac")) formats = ((".ogg", "libopus"), (".mp4", "aac"))
# This test synthesizes .ogg/.mp4 sources itself, so it needs ffmpeg WITH
# the opus/aac encoders. Runners without them (the CI pod) skip; runners
# with the binaries keep full enforcement.
if shutil.which("ffmpeg") is None:
pytest.skip("ffmpeg is unavailable on this runner")
for _suffix, codec in formats:
encoder_probe = subprocess.run(
[
"ffmpeg",
"-v",
"error",
"-f",
"lavfi",
"-i",
"sine=frequency=440:duration=0.1",
"-c:a",
codec,
"-f",
"null",
"-",
],
check=False,
capture_output=True,
text=True,
timeout=15,
)
if encoder_probe.returncode != 0:
pytest.skip(f"ffmpeg lacks the {codec} encoder on this runner")
for suffix, codec in formats: for suffix, codec in formats:
source = tmp_path / f"source{suffix}" source = tmp_path / f"source{suffix}"
subprocess.run( subprocess.run(

View File

@ -58,10 +58,15 @@ def test_thinking_pause_after_first_word_does_not_split(probe_results):
def test_barge_in_stitches_regardless_of_partial_assistant_output(probe_results): def test_barge_in_stitches_regardless_of_partial_assistant_output(probe_results):
scenario = probe_results["second_utterance_during_response_is_complete"] scenario = probe_results["second_utterance_during_response_is_complete"]
assert scenario["firstSends"] == ["alpha"] assert scenario["firstSends"] == ["alpha"]
# Assistant text was already visible when the user talked over the # Assistant text was already visible (and being spoken) when the user
# response: the interrupted thought and the follow-up still form one # talked over the response: the interrupted thought and the follow-up form
# stitched message. # one stitched message, and the cut marker names the sentence that was
assert scenario["sends"] == ["alpha", "alpha bravo charlie"] # playing so the model knows where its reply stopped being heard.
assert scenario["sends"] == [
"alpha",
"alpha bravo charlie\n[voice interruption: you were cut off after "
'"Partial answer already visible."]',
]
def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results): def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results):
@ -124,11 +129,15 @@ def test_overlay_source_contract():
def test_endpointing_and_streaming_hardening_source_contract(): def test_endpointing_and_streaming_hardening_source_contract():
source = VOICE_SCRIPT.read_text(encoding="utf-8") source = VOICE_SCRIPT.read_text(encoding="utf-8")
# Young utterances hold their endpoint past a thinking pause. # Young utterances hold their endpoint past a thinking pause — unless the
# streaming partial already reads as a plausibly complete utterance
# (>=3 words or terminal punctuation), which endpoints at the base window.
assert "const VAD_EARLY_SILENCE_MS=1800" in source assert "const VAD_EARLY_SILENCE_MS=1800" in source
assert "const VAD_COMMITTED_SPEECH_MS=1200" in source assert "const VAD_COMMITTED_SPEECH_MS=1200" in source
assert "latestPartial:function(){return lastPartialText;}" in source
assert "const partialComplete=partialWords>=3||(partialWords>0" in source
assert ( assert (
"const endpointSilenceMs=speechMs<VAD_COMMITTED_SPEECH_MS" "const endpointSilenceMs=(speechMs<VAD_COMMITTED_SPEECH_MS&&!partialComplete)"
"?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs" in source "?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs" in source
) )
# Resume always reaches the wire after a server-length silence gap. # Resume always reaches the wire after a server-length silence gap.
@ -150,9 +159,106 @@ def test_stitch_gate_ignores_partial_assistant_output():
"async function sendTranscript", 1 "async function sendTranscript", 1
)[0] )[0]
assert "currentAssistantText()" not in region assert "currentAssistantText()" not in region
assert "pendingStitch={text:sent.text,at:Date.now()}" in region assert "pendingStitch={text:sent.text,at:Date.now(),cut:cut}" in region
# Normal completion still clears the stitch context. # Normal completion still clears the stitch context.
assert ( assert (
"if(isFinal){\n // The completion callback marks a normally " "if(isFinal){\n // The completion callback marks a normally "
"completed response" in source "completed response" in source
) )
def test_three_word_partial_endpoints_at_base_silence(probe_results):
scenario = probe_results["three_word_partial_endpoints_at_base_silence"]
assert scenario["sends"] == ["alpha bravo charlie"]
assert scenario["state"] == "thinking"
def test_two_word_young_utterance_keeps_the_hold(probe_results):
scenario = probe_results["two_word_young_utterance_keeps_the_hold"]
assert scenario["sendsEarly"] == []
assert scenario["sends"] == ["alpha bravo"]
def test_errored_turn_is_never_spoken_and_capture_resyncs(probe_results):
scenario = probe_results["errored_turn_is_not_spoken_and_capture_resyncs"]
assert scenario["stateAfterError"] == "listening"
assert scenario["labelAfterError"] == "Something went wrong — listening"
assert scenario["ttsDuringError"] == 0
assert scenario["freshSessions"] >= 1
# The follow-up utterance is sent alone: no stitch with the errored turn.
assert scenario["sends"] == ["alpha", "bravo"]
def test_barge_cut_marker_is_one_bounded_line(probe_results):
scenario = probe_results["barge_cut_marker_records_spoken_tail"]
assert scenario["ttsTexts"][0] == "The first point is ready."
assert scenario["sends"][1] == (
"alpha bravo\n[voice interruption: you were cut off after "
'"The first point is ready."]'
)
marker = scenario["sends"][1].split("\n", 1)[1]
assert "\n" not in marker
def test_sticky_language_biases_next_stt_session(probe_results):
scenario = probe_results["sticky_language_biases_next_stt_session"]
assert scenario["startLanguages"][0] == "auto"
assert "en" in scenario["startLanguages"]
def test_cut_marker_error_resync_and_speaking_cycles_source_contract():
source = VOICE_SCRIPT.read_text(encoding="utf-8")
# Cut marker: exactly one appended line, tail bounded to 120 characters.
assert "function voiceCutMarker(cut)" in source
assert "tail.length>120?tail.slice(tail.length-120):tail" in source
assert '\\n[voice interruption: you were cut off after "' in source
# Error envelopes are detected structurally and trigger a capture resync.
assert "function handleAssistantResponseError(token)" in source
assert "function resyncCapture(token,statusLabel)" in source
assert "'Something went wrong — listening'" in source
assert "segment.dataset.error==='1'" in source
assert ".provider-error-details" in source
# A cancellation with no in-flight stream never gates the send.
assert "if(!cancellation.streamId){" in source
# Speaking ⇄ thinking cycles inside one interim-message turn.
assert "function scheduleSpeakingIdleFallback(turn,session)" in source
assert "if(state==='speaking') setState('thinking')" in source
assert "function playbackAudible()" in source
# Body-only scraping: never the avatar letter, author name or status chip.
assert "querySelectorAll('.msg-body')" in source
assert "function readSegmentBody(segment)" in source
assert "function readAssistantTurn(turn)" in source
def test_caption_regions_are_bounded_scrollable_and_follow_tail():
source = VOICE_SCRIPT.read_text(encoding="utf-8")
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(
encoding="utf-8"
)
assert "function updateCaptionRegion(element,text,limit)" in source
assert "function attachCaptionScroll(element)" in source
assert "element.dataset.follow=gap<=24?'1':'0'" in source
for token in (
"overflow-y: auto",
"overscroll-behavior: contain",
"-webkit-overflow-scrolling: touch",
"touch-action: pan-y",
"max-height: 22vh",
"max-height: 38vh",
"mask-image",
):
assert token in css
def test_reply_language_stickiness_source_contract():
source = VOICE_SCRIPT.read_text(encoding="utf-8")
assert "let sessionLanguage=''" in source
assert "language:sessionLanguage||'auto'" in source
assert "function strongReplyLanguage(text)" in source
assert "function detectReplyLanguage(text)" in source
assert "scheduleThinkingCues(token,language||sessionLanguage,thinkingTurnId)" in source
# The sticky language resets with each hands-free session.
assert source.count("sessionLanguage='';") >= 2

View File

@ -134,3 +134,15 @@ def test_stt_client_queue_and_canonical_archive_are_strictly_bounded():
assert "takeFallbackBlob:function()" in source assert "takeFallbackBlob:function()" in source
assert "pcm16WavBlob(archive,16000)" in source assert "pcm16WavBlob(archive,16000)" in source
assert "if(pcmFallback)" in source assert "if(pcmFallback)" in source
def test_playback_audibility_and_settle_short_circuit():
source = VOICE.read_text(encoding="utf-8")
# Barge detection keys off actually audible playback, so the persistent
# PCM worklet node between speech segments no longer counts as speaking.
assert "function playbackAudible()" in source
assert "const playbackActive=playbackAudible();" in source
assert "const playbackLive=playbackAudible();" in source
# A cancellation record with no in-flight stream can never gate the send.
assert "if(!cancellation.streamId){" in source

View File

@ -9,6 +9,8 @@ import shutil
import subprocess import subprocess
import sys import sys
from hux_node_gate import require_node
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
FIXTURE = ROOT / "testing/fixtures/hermes-webui-0.52.181" FIXTURE = ROOT / "testing/fixtures/hermes-webui-0.52.181"
@ -142,6 +144,7 @@ def test_visual_states_have_distinct_layers_finite_error_and_reduced_motion():
def test_dom_probe_exercises_actual_injected_voice_script(): def test_dom_probe_exercises_actual_injected_voice_script():
require_node()
result = subprocess.run( result = subprocess.run(
["node", str(DOM_PROBE), str(VOICE_JS), str(MEDIARECORDER_FIXTURE)], ["node", str(DOM_PROBE), str(VOICE_JS), str(MEDIARECORDER_FIXTURE)],
cwd=ROOT, cwd=ROOT,
@ -207,3 +210,22 @@ def test_release_candidate_keeps_finalized_webm_as_quality_fallback():
assert "new Blob(chunks,{type:recordedMime||'audio/webm'})" in script assert "new Blob(chunks,{type:recordedMime||'audio/webm'})" in script
assert "transcribeStreamingOrFallback" in script assert "transcribeStreamingOrFallback" in script
assert "transcribe(blob,token,turnId)" in script assert "transcribe(blob,token,turnId)" in script
def test_session_continuity_profile_gate_survives_transient_lookup_failures(
tmp_path,
):
"""The /api/session profile gate must not 409 ("This session is
unavailable to this account.") on a cold or transiently failing
root-alias lookup only on a confirmed cross-profile mismatch."""
target = _patched_fixture(tmp_path)
profiles = (target / "api/profiles.py").read_text(encoding="utf-8")
assert "_root_profile_names_confirmed" in profiles
assert (
'if "default" in (row, active) and not _root_profile_names_confirmed():'
in profiles
)
# Listing failure now answers from the last known alias set (one extra
# cache read next to the two pinned upstream reads).
assert profiles.count("return name in _root_profile_name_cache") == 3

View File

@ -868,3 +868,20 @@ def test_notes_document_the_stt_driven_voice_selection_and_its_limits():
assert "STT-detected language" in notes assert "STT-detected language" in notes
for marker in ("hands-free", "Typed messages", "en_US-amy-medium"): for marker in ("hands-free", "Typed messages", "en_US-amy-medium"):
assert marker in notes assert marker in notes
def test_reply_language_heuristic_routes_detectionless_spanish(voice_probe):
"""A Spanish reply on a detection-less turn still gets the Spanish voice."""
requests = voice_probe["spanish_reply_without_detection_uses_reply_heuristic"]["tts"]
assert requests, "voice mode never reached /api/tts"
for request in requests:
assert request["language"] == "es"
def test_reply_script_evidence_corrects_wrong_detection(voice_probe):
"""Script-level evidence in the reply text overrides a wrong STT hint, so
a Spanish reply is never spoken by the English Amy voice."""
requests = voice_probe["reply_script_evidence_corrects_wrong_detection"]["tts"]
assert requests, "voice mode never reached /api/tts"
for request in requests:
assert request["language"] == "es"

View File

@ -395,3 +395,62 @@ test('coordinator default factory and polling callback remain generation fenced'
let staleDestroyed = 0; resolveRuntime({destroy() { staleDestroyed += 1; }}); let staleDestroyed = 0; resolveRuntime({destroy() { staleDestroyed += 1; }});
assert.equal(await first, false); assert.equal(staleDestroyed, 1); stale.destroy(); assert.equal(await first, false); assert.equal(staleDestroyed, 1); stale.destroy();
}); });
test('chrome mounts an icon-only rail toggle beside the app tabs when a rail exists', () => {
const doc = new FakeDocument();
const rail = new FakeNode('nav');
rail.setAttribute('class', 'rail');
const spacer = new FakeNode('div');
spacer.setAttribute('class', 'rail-spacer');
rail.appendChild(spacer);
rail.querySelector = (selector) => (selector === '.rail-spacer' ? spacer : null);
rail.insertBefore = (child, before) => {
const index = rail.children.indexOf(before);
rail.children.splice(index < 0 ? rail.children.length : index, 0, child);
child.parentNode = rail;
};
doc.querySelector = (selector) => (selector === 'nav.rail' ? rail : null);
doc.createElementNS = () => { throw new Error('no namespaces here'); };
const chrome = api.createChrome(doc);
assert.ok(chrome.railToggle);
assert.equal(chrome.railToggle.getAttribute('data-tooltip'), 'Workspace');
assert.equal(chrome.railToggle.getAttribute('aria-label'), 'Workspace');
assert.equal(chrome.railToggle.getAttribute('aria-controls'), 'huxWorkspaceDrawer');
assert.match(chrome.railToggle.attributes.class, /rail-btn/);
assert.equal(chrome.railToggle.textContent, '⧉');
assert.equal(rail.children[0], chrome.railToggle);
assert.equal(rail.children[1], spacer);
assert.match(chrome.open.attributes.class, /--railed/);
chrome.railToggle.trigger('click');
assert.equal(chrome.drawer.hidden, false);
assert.equal(chrome.railToggle.getAttribute('aria-expanded'), 'true');
assert.equal(chrome.open.getAttribute('aria-expanded'), 'true');
chrome.setOpen(false);
assert.equal(chrome.railToggle.getAttribute('aria-expanded'), 'false');
chrome.destroy();
assert.equal(rail.children.includes(chrome.railToggle), false);
assert.equal(doc.body.children.length, 0);
});
test('rail toggle renders an svg icon where namespaces exist and falls back floating without a rail', () => {
const doc = new FakeDocument();
const rail = new FakeNode('nav');
rail.querySelector = () => null; // no spacer: toggle appends at the end
rail.insertBefore = (child) => { rail.children.push(child); child.parentNode = rail; };
doc.querySelector = (selector) => (selector === 'nav.rail' ? rail : null);
doc.createElementNS = (_ns, tag) => new FakeNode(tag);
const chrome = api.createChrome(doc);
assert.equal(chrome.railToggle.children.length, 1);
assert.equal(chrome.railToggle.children[0].tagName, 'SVG');
assert.equal(chrome.railToggle.children[0].children[0].tagName, 'PATH');
chrome.destroy();
// Rail lookups that yield nothing usable keep the plain floating toggle.
const bare = new FakeDocument();
bare.querySelector = () => new FakeNode('nav'); // no insertBefore
const floating = api.createChrome(bare);
assert.equal(floating.railToggle, null);
assert.equal(floating.open.attributes.class, 'hux-workspace-toggle');
assert.equal(floating.open.textContent, 'Workspace');
floating.destroy();
});