b40300efaf
The 'Something went wrong - listening' state with no spoken answer was a false positive: readAssistantTurn flagged the whole turn as an error if ANY segment was error-stamped - including a recovered/transient tool error or a cancellation notice from an earlier interim - and threw away the real answer that the same turn produced. Error now surfaces only when the turn yielded no spoken answer at all; a turn with real content is spoken normally. Softened the genuine-error label to the friendlier 'Let's try that again - listening'. New probe scenarios lock both: an error segment alongside an answer speaks the answer (error=false), and an error-only turn still reports the error. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
2727 lines
124 KiB
JavaScript
2727 lines
124 KiB
JavaScript
// Natural turn-taking for chat.bstein.dev using the private Jetsons.
|
||
(function(){
|
||
'use strict';
|
||
|
||
const modeBtn=document.getElementById('btnVoiceMode');
|
||
const bar=document.getElementById('voiceModeBar');
|
||
const indicator=document.getElementById('voiceModeIndicator');
|
||
const label=document.getElementById('voiceModeLabel');
|
||
const composer=document.getElementById('msg');
|
||
if(!modeBtn||!bar||!indicator||!label||!composer||!navigator.mediaDevices||!window.MediaRecorder) return;
|
||
|
||
let ready=false;
|
||
let active=false;
|
||
let state='idle';
|
||
let generation=0;
|
||
let turnSequence=0;
|
||
let recorder=null;
|
||
let stream=null;
|
||
let audioContext=null;
|
||
let captureNode=null;
|
||
let vadTimer=null;
|
||
let responsePollTimer=null;
|
||
let currentAudio=null;
|
||
let playbackSession=null;
|
||
let bargeMonitor=null;
|
||
let bargeCancelPromise=null;
|
||
let suppressAutoRead=false;
|
||
let thinkingCue=null;
|
||
let streamingStt=null;
|
||
let captureTurnId='';
|
||
let thinkingTurnId='';
|
||
let thinkingSession=null;
|
||
let assistantBaseline=null;
|
||
let speechTurn=null;
|
||
let errorTimer=null;
|
||
let visualInputLevel=0;
|
||
let streamingCapability={tts:null,stt:null,preflight:null};
|
||
let voicePreflight=null;
|
||
// Continuous-capture state. The microphone stream and its AudioContext stay
|
||
// hot for the whole hands-free session; each utterance is one "capture turn"
|
||
// guarded by captureGeneration so a response-side barge (which bumps
|
||
// `generation`) never interrupts the running microphone pipeline.
|
||
let captureGeneration=0;
|
||
let captureActive=false;
|
||
let captureGraph=null;
|
||
// Barge-in stitching: transcript whose model response was cancelled before
|
||
// any visible assistant output, plus the transcript most recently sent.
|
||
let pendingStitch=null;
|
||
let lastSentTranscript=null;
|
||
const voiceTabNonce=(function(){
|
||
try{
|
||
const bytes=new Uint8Array(16);
|
||
window.crypto.getRandomValues(bytes);
|
||
return Array.from(bytes,function(value){return value.toString(16).padStart(2,'0');}).join('');
|
||
}catch(_){return '';}
|
||
})();
|
||
const reducedMotion=window.matchMedia?window.matchMedia('(prefers-reduced-motion: reduce)'):{matches:false};
|
||
// FIX 2: the app's own Hermes caduceus mark (static/favicon.svg), embedded
|
||
// as a centred watermark inside the conversation orb. Rendered monochrome via
|
||
// currentColor at low opacity so it reads across every state tint
|
||
// (idle/listening/transcribing/thinking/speaking) without ever obscuring the
|
||
// energy animation, and it carries no animation of its own (reduced-motion safe).
|
||
const HERMES_MARK_SVG='<svg viewBox="0 0 64 64" xmlns="http://www.w3.org/2000/svg" focusable="false" aria-hidden="true"><g transform="translate(-12.326 -15.707) scale(0.07072)"><path fill="currentColor" fill-rule="evenodd" d="M630.5 961.9 C634.9 960.7 638.5 957.9 640.5 953.9 C642.5 950.1 643.3 865.1 641.4 864.3 C640 863.8 623.9 872.5 618.2 876.8 C616.4 878.2 613.8 881.2 612.5 883.5 L610 887.7 610 918.4 C610 951.8 610.2 953.1 615.7 958.3 C618.3 960.8 622.4 962.7 625.5 962.9 C626 963 628.3 962.5 630.5 961.9 Z M596 913 C596.8 911.5 596.6 909.4 595.4 904.8 C592.1 892.1 595.4 881.4 605.5 872.1 C612.9 865.4 621.2 860.6 641.1 851.4 C681.3 832.9 691.1 827.1 704.5 813.6 C724.9 793.1 730 768.6 718.9 745.5 C714.9 737.4 705.5 727.4 696.7 722.1 L691.1 718.8 678.3 722.6 C671.3 724.6 664.9 726.8 664.2 727.3 C663.5 727.9 663 730.6 663 734.1 C663 739.9 663.1 740 666.8 741.9 C672.9 745 680.6 752.6 683.4 758.2 C688.9 769.3 686 781.3 675.3 791 C666.4 799.1 662.1 801.7 631.3 817.2 C598.7 833.5 587.2 840.5 578.9 849 C565.9 862.3 561.8 880.1 568.3 894.4 C574.4 907.7 592.4 919.8 596 913 Z M579.8 832.2 C582.7 830.2 586.8 827.3 589 825.8 C592.9 823.1 593 822.9 593 817.2 L593 811.4 586.6 807.2 C578.4 801.8 572.5 795.2 568.7 787.2 C566 781.5 565.8 780.1 566.2 773.5 C566.8 764.5 569.4 759.6 577.8 751.8 C589.1 741.4 603.5 735.3 666 714.8 C687.7 707.6 710.2 699.7 715.9 697.2 C741.9 685.8 757.8 670 764.5 648.7 C765.9 644.4 767 639.2 767 637.1 C767 631.5 768.1 631 777.9 631.6 C801.3 633.2 819.4 623.2 829 603.6 C831.2 599.2 833.5 593.4 834.2 590.7 C835.3 586.4 835.2 585.8 833.4 584 C831.5 582.1 830.8 582 814 583 C804.4 583.5 789.8 584.1 781.5 584.2 L766.5 584.5 766.2 577.9 C766 573.4 766.3 571 767.2 570.2 C767.9 569.6 778.4 568.4 790.4 567.6 C835.7 564.3 849.7 561.9 862.2 555.3 C878.5 546.7 889.5 529.3 893 506.4 C894.1 499.3 894 498.6 892.3 496.8 C890.4 494.9 890 495 865.4 500.4 C838.7 506.3 789.4 516.1 776.8 518.1 C766.3 519.7 766 519.5 766 511.2 C766 507.2 766.5 503.7 767.2 502.8 C767.9 502 771.2 500.7 774.5 500.1 C788.7 497.1 852.9 480.7 868.5 476 C877.9 473.2 889.8 468.8 895 466.3 C903 462.5 905.8 460.4 913.1 453.1 C922.9 443.2 928.3 433.9 932.5 419.5 C935.4 409.5 937.6 393.5 936.6 389.7 C935.3 384.3 933.5 384.5 914.3 392.1 C879.6 406 825.4 423.1 754.6 442.4 C728.5 449.6 719.1 452.5 717.2 454.2 C711.9 459.2 712 457.7 712 516.7 C712 551.6 711.6 572.9 710.9 575.2 C709.6 580 703.8 585.7 699.1 587 C697.1 587.5 689.9 588.2 683 588.6 C674 589.1 670 589.7 668.8 590.8 C667.2 592.1 667 594.3 667 608 C667 621.6 667.2 624.1 668.8 625.8 C670.4 627.8 671.8 627.9 694.9 628.2 C710.4 628.4 719.8 628.9 720.8 629.6 C723.1 631.2 720.6 639.8 715.9 646.9 C706.4 661.1 694.2 667.1 631 689 C581.4 706.1 563.9 713.8 550.3 724.7 C512.3 755 518.4 806 563.1 831.3 C567.7 833.9 572.2 836 573.1 836 C573.9 836 577 834.3 579.8 832.2 Z M626.3 807.4 C633.5 803.8 640.2 800.1 641.1 799.4 C642.5 798.1 642.7 794.2 642.5 766.5 C642.5 749.2 642.1 734.7 641.7 734.4 C641 733.7 613.6 743.6 611.2 745.3 C610.3 746 610 754.1 610 779.5 C610 797.7 610.3 813 610.7 813.3 C611.8 814.5 612.7 814.1 626.3 807.4 Z M571.1 699.2 L584.5 693.4 584.8 685.5 C585 681.2 584.7 677.3 584.1 676.7 C583.6 676.2 579.9 674.7 575.8 673.3 C567.5 670.5 554.7 664.2 548.4 660 C538.6 653.2 530.5 640.3 531.2 632.6 L531.5 629.5 541 628.9 C546.2 628.5 557.7 628.2 566.5 628.1 C577.6 628 583 627.6 583.8 626.8 C585.5 625.1 585.5 591.6 583.8 590.3 C583.1 589.7 576.5 589 569.1 588.6 C555.2 587.9 550.4 586.7 546.1 582.7 C541 578 541 578.1 541 518.8 C541 466.4 540.9 463.3 539 459.3 C536.5 453.7 533.3 451.9 518.7 448.1 C442.4 428 363.2 402.9 330.3 388.3 C322.4 384.9 322 384.8 319.5 386.4 C317.3 387.9 317 388.7 317 393.9 C317 397.1 317.7 403.5 318.5 408.1 C324.3 439.3 338.4 458 364.5 469.2 C374.6 473.5 396.4 479.7 441.3 491 C464.8 496.9 484.7 502.3 485.5 503 C486.6 503.9 487 506.3 487 511.2 C487 519.5 486.6 519.8 476.7 518.1 C461.5 515.5 412.5 505.6 391.5 500.9 C379.4 498.2 368.3 495.7 366.7 495.3 C364.8 494.9 363.3 495.3 361.9 496.6 C360 498.3 359.9 499.2 360.5 505.4 C362.5 526 373.6 544.9 388.9 554 C401.6 561.5 417.9 564.4 468 568.1 C477.1 568.7 485.1 569.7 485.8 570.3 C487.5 571.7 487.4 582.4 485.6 583.9 C484.2 585.1 473.4 584.7 432.5 582.4 C422.2 581.8 421.4 581.9 419.7 583.8 C417.9 585.8 417.9 586.1 419.5 591.6 C424.1 607.4 434.2 620.5 446.4 626.5 C455.1 630.8 461.3 632 474.1 632 C479.5 632 484.1 632.4 484.4 632.9 C484.8 633.4 485.6 637.4 486.4 641.7 C489.7 660.4 500.4 676.9 516.5 688 C526.3 694.7 549.1 704.8 555.1 704.9 C556.5 705 563.7 702.4 571.1 699.2 Z M628.9 678.9 C635.3 676.6 641 674.2 641.5 673.5 C643.1 671.6 642.5 576.3 640.9 574.4 C639.4 572.5 613.1 572.3 611.2 574.2 C610.3 575.1 610 588.7 610 629 C610 658.5 610.3 683 610.7 683.4 C611.8 684.5 616 683.5 628.9 678.9 Z M636.1 556 C654.1 552.6 669.6 536.9 673.6 517.8 C677.9 497.7 668.2 476.3 650 465.8 C636.2 457.8 615.8 458 601.7 466.3 C595.3 470.1 586.1 480.5 582.7 487.8 C570 514.9 585.4 547.4 614.7 555.5 C620.6 557.1 629 557.3 636.1 556 Z"/></g></svg>';
|
||
const ERROR_VISIBLE_MS=3200;
|
||
const STREAMING_CAPABILITY_URL='/api/voice/streaming/capability';
|
||
const TTS_STREAM_URL='/api/tts/stream';
|
||
const STT_STREAM_PATH='/api/transcribe/stream';
|
||
const VOICE_PREFLIGHT_URL='/api/voice/route-preflight';
|
||
const VOICE_PREFLIGHT_DEBOUNCE_MS=200;
|
||
const VOICE_PREFLIGHT_DEADLINE_MS=1100;
|
||
const VOICE_PREFLIGHT_TIERS=['fast','balanced','deep','maximum'];
|
||
const WORKLET_URL='static/atlas-voice-worklet.js';
|
||
const STT_MAX_QUEUED_BYTES=1048576;
|
||
const STT_MAX_ARCHIVE_BYTES=2880000;
|
||
const BARGE_LOOKBACK_MS=900;
|
||
const BARGE_DUCK_FRAMES=2;
|
||
const BARGE_TRIGGER_FRAMES=4;
|
||
// A response cancelled by barge-in leaves the user mid-thought: the next
|
||
// finalized utterance inside this window is sent as one stitched message.
|
||
const STITCH_WINDOW_MS=20000;
|
||
// Endpointing. A short utterance ("The…") is usually a sentence still being
|
||
// formed, so while less than VAD_COMMITTED_SPEECH_MS of voiced audio has
|
||
// been collected the endpoint silence window stretches to
|
||
// VAD_EARLY_SILENCE_MS: a thinking pause right after the first word can no
|
||
// longer commit a one-word fragment of the sentence.
|
||
const VAD_EARLY_SILENCE_MS=1800;
|
||
const VAD_COMMITTED_SPEECH_MS=1200;
|
||
// The private STT server freezes a speculative end-of-speech snapshot after
|
||
// 650ms of server-side silence even when this client never sent speculate.
|
||
// Speech resuming after a gap that long always sends an explicit resume so
|
||
// a frozen first-word snapshot can never survive into the commit.
|
||
const SERVER_EOS_SILENCE_MS=650;
|
||
// Never speculate on the first inter-word gap of a young utterance: a
|
||
// wasted Whisper pass over one word occupies the Jetson for seconds and
|
||
// stalls the real commit that follows (observed live as multi-second
|
||
// commit pending waits).
|
||
const SPECULATE_MIN_SPEECH_MS=700;
|
||
const TTS_SPEED_DEFAULT=1.15;
|
||
const TTS_SPEED_MIN=0.5;
|
||
const TTS_SPEED_MAX=2;
|
||
const THINKING_CUE_FIRST_MS=1900;
|
||
const THINKING_CUE_INTERVAL_MS=6500;
|
||
const THINKING_CUE_POOLS={
|
||
en:[{id:'thinking',text:"I'm thinking."},{id:'let_me_think',text:'Let me think.'},{id:'still_working',text:'Still working on that.'},{id:'one_more_moment',text:'One more moment.'}],
|
||
ru:[{id:'thinking',text:'Я думаю.'},{id:'let_me_think',text:'Дайте подумать.'},{id:'still_working',text:'Я всё ещё думаю над этим.'},{id:'one_more_moment',text:'Ещё мгновение.'}],
|
||
es:[{id:'thinking',text:'Estoy pensando.'},{id:'let_me_think',text:'Déjame pensar.'},{id:'still_working',text:'Sigo pensando en eso.'},{id:'one_more_moment',text:'Un momento más.'}],
|
||
};
|
||
const STATE_LABELS={
|
||
listening:'Listening',
|
||
transcribing:'Transcribing…',
|
||
thinking:'Thinking…',
|
||
speaking:'Speaking',
|
||
error:'Voice unavailable',
|
||
idle:'',
|
||
};
|
||
// The only language signal this file trusts is the one the private Whisper
|
||
// service returned for the audio of the turn currently being answered. It is
|
||
// bound to that turn's generation token and consumed exactly once.
|
||
let sttLanguage='';
|
||
let sttLanguageToken=-1;
|
||
// Sticky per-hands-free-session language: set by the private Whisper
|
||
// detection of the user's own speech (or a strong reply-text signal on a
|
||
// detection-less turn), consumed as the NEXT streaming STT session's bias
|
||
// and as the thinking-cue locale. Cleared on activate/deactivate.
|
||
let sessionLanguage='';
|
||
// Forced conversation-mode language (FIX 3): when non-empty it overrides both
|
||
// the streaming STT language hint and the reply-TTS voice, superseding
|
||
// auto-detection until the user picks Auto again. Session-scoped; no storage.
|
||
let forcedLanguage='';
|
||
// Bounded retry budget for the completion (isFinal) pump: the STREAM_DONE
|
||
// callback can beat the settle re-render that stamps the answer onto
|
||
// data-raw-text, so a transient empty read must never tear a live reply down.
|
||
let finalizeAttempts=0;
|
||
const TTS_LANGUAGES=['en','ru','es'];
|
||
// FIX 3: conversation-mode language chooser. Auto (default) plus the languages
|
||
// the private voice map supports — kept in lockstep with
|
||
// dockerfiles/hermes-jetson-tts-server.py LANGUAGE_VOICE_MAP (en → amy,
|
||
// es → claude, ru → irina). Selecting one FORCES both the reply TTS voice and
|
||
// the streaming STT language hint for the rest of the hands-free session (via
|
||
// forcedLanguage), overriding auto-detection until Auto is chosen again.
|
||
const CONVERSATION_LANGUAGES=[
|
||
{code:'',label:'Auto',sublabel:'Detect'},
|
||
{code:'en',label:'English',sublabel:'English'},
|
||
{code:'es',label:'Español',sublabel:'Spanish'},
|
||
{code:'ru',label:'Русский',sublabel:'Russian'},
|
||
];
|
||
const GLOBE_ICON_SVG='<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.5" aria-hidden="true" focusable="false"><circle cx="10" cy="10" r="7.25"/><path d="M2.9 10h14.2M10 2.75c1.9 2 2.9 4.6 2.9 7.25S11.9 15.25 10 17.25C8.1 15.25 7.1 12.65 7.1 10S8.1 4.75 10 2.75z"/></svg>';
|
||
const originalAutoRead=window.autoReadLastAssistant;
|
||
const originalApplyPreference=window._applyVoiceModePref;
|
||
|
||
function normalizeSttLanguage(value){
|
||
if(typeof value!=='string') return '';
|
||
const code=value.trim().toLowerCase();
|
||
return TTS_LANGUAGES.indexOf(code)>=0?code:'';
|
||
}
|
||
|
||
function clearSttLanguage(){
|
||
sttLanguage='';
|
||
sttLanguageToken=-1;
|
||
}
|
||
|
||
function rememberSttLanguage(language,token){
|
||
sttLanguage=language||'';
|
||
sttLanguageToken=sttLanguage?token:-1;
|
||
}
|
||
|
||
function takeSttLanguage(token){
|
||
const language=sttLanguageToken===token?sttLanguage:'';
|
||
clearSttLanguage();
|
||
return language;
|
||
}
|
||
|
||
const SPANISH_ORTHOGRAPHY=/[áéíóúñü¡¿]/i;
|
||
const SPANISH_STOPWORDS=/\b(?:el|la|los|las|un|una|es|está|qué|para|por|con|pero|como|más|sí|gracias|hola|puedo|también|muy|este|esta|todo|bien)\b/g;
|
||
|
||
function strongReplyLanguage(text){
|
||
// Script-level certainty only: Cyrillic text is Russian; Spanish
|
||
// orthography (accents, ñ, inverted punctuation) is Spanish. Plain-ASCII
|
||
// text yields no signal, so an English reply never flips a trusted
|
||
// STT-detected voice.
|
||
const sample=String(text||'').slice(0,400);
|
||
if(/[Ѐ-ӿ]/.test(sample)) return 'ru';
|
||
if(SPANISH_ORTHOGRAPHY.test(sample)) return 'es';
|
||
return '';
|
||
}
|
||
|
||
function detectReplyLanguage(text){
|
||
// Lightweight reply-language heuristic for turns without a trusted STT
|
||
// detection: script evidence first, then Spanish stopword density (an
|
||
// accent-free Spanish sentence still routes to the Spanish voice).
|
||
// Returns '' for English/unknown, which the private TTS service resolves
|
||
// to its own English default voice.
|
||
const strong=strongReplyLanguage(text);
|
||
if(strong) return strong;
|
||
const sample=String(text||'').slice(0,400).toLowerCase();
|
||
const words=sample.split(/\s+/).filter(Boolean);
|
||
if(words.length>=4){
|
||
const matches=(sample.match(SPANISH_STOPWORDS)||[]).length;
|
||
if(matches>=2&&matches/words.length>=0.12) return 'es';
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function toast(message){
|
||
if(typeof window.showToast==='function') window.showToast(message,3000);
|
||
}
|
||
|
||
function setState(next,customLabel){
|
||
state=next;
|
||
indicator.className='voice-mode-indicator '+next;
|
||
bar.dataset.voiceState=next;
|
||
bar.setAttribute('aria-busy',next==='transcribing'||next==='thinking'?'true':'false');
|
||
label.textContent=customLabel||STATE_LABELS[next]||'';
|
||
bar.style.display=(active&&next!=='idle')||next==='error'?'':'none';
|
||
resetInputLevel();
|
||
syncConversationOverlay(next,customLabel);
|
||
}
|
||
|
||
function resetInputLevel(){
|
||
visualInputLevel=0;
|
||
indicator.style.setProperty('--voice-ripple-scale','1.035');
|
||
indicator.style.setProperty('--voice-ripple-opacity','0.3');
|
||
}
|
||
|
||
function updateInputLevel(rms){
|
||
if(reducedMotion.matches||state!=='listening') return;
|
||
const target=Math.max(0,Math.min(1,(rms-0.01)/0.18));
|
||
visualInputLevel=(visualInputLevel*0.72)+(target*0.28);
|
||
indicator.style.setProperty('--voice-ripple-scale',(1.035+(visualInputLevel*0.16)).toFixed(3));
|
||
indicator.style.setProperty('--voice-ripple-opacity',(0.26+(visualInputLevel*0.48)).toFixed(3));
|
||
if(conversation) conversation.root.style.setProperty('--conversation-level',visualInputLevel.toFixed(3));
|
||
}
|
||
|
||
// ── Conversation mode overlay ─────────────────────────────────────────
|
||
// Hands-free is a full-screen conversation: a breathing orb that follows
|
||
// microphone energy while listening and TTS playback while speaking, the
|
||
// live state caption, streaming user/assistant captions, and mute/exit
|
||
// controls. The overlay is created lazily on activation, removed completely
|
||
// on deactivate, keeps no storage, and is silently skipped in environments
|
||
// without a usable DOM (headless contract probes).
|
||
let conversation=null;
|
||
// FIX 4: when the model has been thinking for a beat with no reply text yet,
|
||
// the overlay shows a subtle animated "working…" affordance so the dead time
|
||
// reads as alive rather than frozen. Purely a client progress hint — it never
|
||
// fabricates spoken acknowledgements (that is the model's job).
|
||
let awaitingTimer=null;
|
||
const AWAITING_AFFORDANCE_MS=2500;
|
||
|
||
function conversationUsable(){
|
||
return !!(typeof document!=='undefined'&&document&&document.body&&typeof document.createElement==='function');
|
||
}
|
||
|
||
function clearAwaitingAffordance(){
|
||
if(awaitingTimer){window.clearTimeout(awaitingTimer);awaitingTimer=null;}
|
||
if(conversation&&conversation.root.classList) conversation.root.classList.remove('is-awaiting');
|
||
}
|
||
|
||
function updateAwaitingAffordance(next){
|
||
// Arm the affordance only while Thinking and only after the grace window;
|
||
// any other state, or the first scrap of reply text, disarms it.
|
||
clearAwaitingAffordance();
|
||
if(!conversation||next!=='thinking') return;
|
||
awaitingTimer=window.setTimeout(function(){
|
||
awaitingTimer=null;
|
||
if(conversation&&conversation.root.dataset.voiceState==='thinking'&&conversation.root.classList){
|
||
conversation.root.classList.add('is-awaiting');
|
||
}
|
||
},AWAITING_AFFORDANCE_MS);
|
||
}
|
||
|
||
function conversationNode(tag,className,attributes){
|
||
const node=document.createElement(tag);
|
||
if(className) node.className=className;
|
||
if(attributes) Object.keys(attributes).forEach(function(name){node.setAttribute(name,attributes[name]);});
|
||
return node;
|
||
}
|
||
|
||
function syncConversationOverlay(next,customLabel){
|
||
if(!conversation) return;
|
||
conversation.root.dataset.voiceState=next;
|
||
conversation.stateEl.textContent=customLabel||STATE_LABELS[next]||'';
|
||
updateAwaitingAffordance(next);
|
||
}
|
||
|
||
function updateCaptionRegion(element,text,limit){
|
||
// Captions are bounded scrollable regions: keep following the streaming
|
||
// tail unless the user scrolled up inside this region (data-follow='0',
|
||
// maintained by the scroll listener installed at overlay build time).
|
||
const value=String(text||'').slice(-limit);
|
||
element.textContent=value;
|
||
if(!value&&element.dataset) element.dataset.follow='1';
|
||
if((!element.dataset||element.dataset.follow!=='0')&&typeof element.scrollHeight==='number'){
|
||
try{element.scrollTop=element.scrollHeight;}catch(_){ }
|
||
}
|
||
}
|
||
|
||
function attachCaptionScroll(element){
|
||
if(!element.addEventListener) return;
|
||
element.addEventListener('scroll',function(){
|
||
const gap=(element.scrollHeight||0)-(element.scrollTop||0)-(element.clientHeight||0);
|
||
if(element.dataset) element.dataset.follow=gap<=24?'1':'0';
|
||
});
|
||
}
|
||
|
||
function setConversationUserCaption(text){
|
||
if(conversation) updateCaptionRegion(conversation.userCaption,text,4000);
|
||
}
|
||
|
||
function setConversationAssistantCaption(text){
|
||
if(!conversation) return;
|
||
// The first scrap of reply text means the model is no longer silently
|
||
// thinking: disarm the "working…" affordance immediately (FIX 4).
|
||
if(text&&String(text).trim()) clearAwaitingAffordance();
|
||
updateCaptionRegion(conversation.assistantCaption,text,9000);
|
||
}
|
||
|
||
function setConversationPlaying(playing){
|
||
if(!conversation) return;
|
||
if(playing) conversation.root.classList.add('is-playing');
|
||
else conversation.root.classList.remove('is-playing');
|
||
}
|
||
|
||
function applyConversationMute(){
|
||
// True microphone mute on the session's single retained getUserMedia
|
||
// stream: disabled tracks deliver silence to the recorder, the VAD and
|
||
// the streaming STT worklet alike. No second capture is ever opened.
|
||
if(!conversation) return;
|
||
try{
|
||
const tracks=stream&&stream.getAudioTracks?stream.getAudioTracks():[];
|
||
tracks.forEach(function(track){track.enabled=!conversation.muted;});
|
||
}catch(_){ }
|
||
}
|
||
|
||
function setConversationMuted(muted){
|
||
if(!conversation) return;
|
||
conversation.muted=muted;
|
||
applyConversationMute();
|
||
try{
|
||
conversation.muteBtn.setAttribute('aria-pressed',muted?'true':'false');
|
||
conversation.muteBtn.textContent=muted?'Unmute microphone':'Mute microphone';
|
||
if(muted) conversation.root.classList.add('is-muted');
|
||
else conversation.root.classList.remove('is-muted');
|
||
}catch(_){ }
|
||
}
|
||
|
||
function conversationKeydown(event){
|
||
if(!conversation) return;
|
||
if(event.key==='Escape'){
|
||
if(event.preventDefault) event.preventDefault();
|
||
// Escape peels one layer at a time: an open language menu closes first
|
||
// (focus returns to its button), and only a second Escape exits the mode.
|
||
if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}
|
||
deactivate(true);
|
||
return;
|
||
}
|
||
if(event.key==='Tab'){
|
||
// Minimal focus trap across the overlay controls (language, mute, exit).
|
||
const stops=[conversation.langBtn,conversation.muteBtn,conversation.exitBtn].filter(Boolean);
|
||
const current=stops.indexOf(document.activeElement);
|
||
const index=current<0?(event.shiftKey?0:stops.length-1):current;
|
||
const next=stops[(index+(event.shiftKey?stops.length-1:1))%stops.length];
|
||
if(event.preventDefault) event.preventDefault();
|
||
if(next&&next.focus) next.focus();
|
||
}
|
||
}
|
||
|
||
function closeLanguageMenu(focusButton){
|
||
if(!conversation||!conversation.langMenu) return;
|
||
conversation.langMenu.hidden=true;
|
||
conversation.langMenu.setAttribute('hidden','');
|
||
if(conversation.langBtn){
|
||
conversation.langBtn.setAttribute('aria-expanded','false');
|
||
if(focusButton&&conversation.langBtn.focus) conversation.langBtn.focus();
|
||
}
|
||
}
|
||
|
||
function openLanguageMenu(){
|
||
if(!conversation||!conversation.langMenu) return;
|
||
conversation.langMenu.hidden=false;
|
||
conversation.langMenu.removeAttribute('hidden');
|
||
if(conversation.langBtn) conversation.langBtn.setAttribute('aria-expanded','true');
|
||
// Focus the currently selected option so keyboard users land on it.
|
||
const items=conversation.langItems||[];
|
||
const active=items.filter(function(item){return item.getAttribute('aria-checked')==='true';})[0]||items[0];
|
||
if(active&&active.focus) active.focus();
|
||
}
|
||
|
||
function toggleLanguageMenu(){
|
||
if(!conversation||!conversation.langMenu) return;
|
||
if(conversation.langMenu.hidden) openLanguageMenu(); else closeLanguageMenu(true);
|
||
}
|
||
|
||
function reflectLanguageSelection(){
|
||
if(!conversation||!conversation.langItems) return;
|
||
conversation.langItems.forEach(function(item){
|
||
const selected=(item.getAttribute('data-lang')||'')===forcedLanguage;
|
||
item.setAttribute('aria-checked',selected?'true':'false');
|
||
});
|
||
if(conversation.langBtn){
|
||
const current=CONVERSATION_LANGUAGES.filter(function(entry){return entry.code===forcedLanguage;})[0];
|
||
const label=current?current.label:'Auto';
|
||
conversation.langBtn.setAttribute('aria-label','Conversation language: '+label);
|
||
conversation.langBtn.setAttribute('title','Language — '+label);
|
||
if(conversation.langBtn.classList){
|
||
if(forcedLanguage) conversation.langBtn.classList.add('is-forced');
|
||
else conversation.langBtn.classList.remove('is-forced');
|
||
}
|
||
}
|
||
}
|
||
|
||
function selectConversationLanguage(code){
|
||
// Force (or, for '', release back to auto-detection) the session language.
|
||
forcedLanguage=code||'';
|
||
reflectLanguageSelection();
|
||
// Re-bias the open streaming STT session immediately when it is safe — i.e.
|
||
// it has heard nothing yet — so the forced hint applies to the very next
|
||
// utterance instead of only the one after it. Mirrors the auto-switch path.
|
||
if(active&&captureActive&&streamingStt&&typeof streamingStt.latestPartial==='function'&&!streamingStt.latestPartial()){
|
||
startListening(generation,{preserveDisplay:true});
|
||
}
|
||
}
|
||
|
||
function buildLanguageControl(){
|
||
const wrap=conversationNode('div','voice-conversation-lang');
|
||
const btn=conversationNode('button','voice-conversation-lang-btn',{
|
||
type:'button','aria-haspopup':'menu','aria-expanded':'false',
|
||
'aria-label':'Conversation language: Auto',title:'Language — Auto',
|
||
});
|
||
btn.innerHTML=GLOBE_ICON_SVG;
|
||
const menu=conversationNode('div','voice-conversation-lang-menu',{role:'menu','aria-label':'Conversation language',hidden:''});
|
||
menu.hidden=true;
|
||
const items=CONVERSATION_LANGUAGES.map(function(entry){
|
||
const item=conversationNode('button','voice-conversation-lang-item',{
|
||
type:'button',role:'menuitemradio','data-lang':entry.code,
|
||
'aria-checked':entry.code===forcedLanguage?'true':'false',
|
||
});
|
||
item.textContent=entry.label;
|
||
item.addEventListener('click',function(){
|
||
selectConversationLanguage(entry.code);
|
||
closeLanguageMenu(true);
|
||
});
|
||
menu.appendChild(item);
|
||
return item;
|
||
});
|
||
btn.addEventListener('click',function(event){
|
||
if(event&&event.stopPropagation) event.stopPropagation();
|
||
toggleLanguageMenu();
|
||
});
|
||
wrap.appendChild(btn);
|
||
wrap.appendChild(menu);
|
||
return {wrap:wrap,btn:btn,menu:menu,items:items};
|
||
}
|
||
|
||
function openConversationOverlay(){
|
||
if(conversation||!conversationUsable()) return;
|
||
try{
|
||
const root=conversationNode('div','voice-conversation',{
|
||
role:'dialog','aria-modal':'true','aria-label':'Voice conversation',tabindex:'-1',
|
||
});
|
||
const orb=conversationNode('div','voice-conversation-orb',{'aria-hidden':'true'});
|
||
orb.appendChild(conversationNode('span','voice-conversation-orb-halo'));
|
||
orb.appendChild(conversationNode('span','voice-conversation-orb-core'));
|
||
orb.appendChild(conversationNode('span','voice-conversation-orb-ring'));
|
||
// FIX 2: Hermes mark watermark, centred in the orb beneath the energy layers.
|
||
const orbMark=conversationNode('span','voice-conversation-orb-mark',{'aria-hidden':'true'});
|
||
orbMark.innerHTML=HERMES_MARK_SVG;
|
||
orb.appendChild(orbMark);
|
||
const stateEl=conversationNode('div','voice-conversation-state',{'aria-hidden':'true'});
|
||
const captions=conversationNode('div','voice-conversation-captions',{'aria-live':'polite'});
|
||
// Each caption is its own bounded, independently scrollable region
|
||
// (keyboard focusable, touch scrollable); streaming keeps following the
|
||
// tail until the user scrolls up inside that region.
|
||
const userCaption=conversationNode('p','voice-conversation-caption-user',{tabindex:'0'});
|
||
const assistantCaption=conversationNode('p','voice-conversation-caption-assistant',{tabindex:'0'});
|
||
attachCaptionScroll(userCaption);
|
||
attachCaptionScroll(assistantCaption);
|
||
captions.appendChild(userCaption);
|
||
captions.appendChild(assistantCaption);
|
||
const controls=conversationNode('div','voice-conversation-controls');
|
||
const muteBtn=conversationNode('button','voice-conversation-mute',{type:'button','aria-pressed':'false'});
|
||
muteBtn.textContent='Mute microphone';
|
||
const exitBtn=conversationNode('button','voice-conversation-exit',{type:'button'});
|
||
exitBtn.textContent='Exit voice mode';
|
||
controls.appendChild(muteBtn);
|
||
controls.appendChild(exitBtn);
|
||
const lang=buildLanguageControl();
|
||
root.appendChild(lang.wrap);
|
||
root.appendChild(orb);
|
||
root.appendChild(stateEl);
|
||
root.appendChild(captions);
|
||
root.appendChild(controls);
|
||
muteBtn.addEventListener('click',function(){if(conversation) setConversationMuted(!conversation.muted);});
|
||
exitBtn.addEventListener('click',function(){deactivate(true);});
|
||
root.addEventListener('keydown',conversationKeydown);
|
||
// A tap anywhere outside the language menu closes it (never deactivates).
|
||
root.addEventListener('click',function(event){
|
||
if(!conversation||!conversation.langMenu||conversation.langMenu.hidden) return;
|
||
const target=event&&event.target;
|
||
const inWrap=target&&typeof target.closest==='function'&&target.closest('.voice-conversation-lang');
|
||
if(!inWrap) closeLanguageMenu(false);
|
||
});
|
||
document.body.appendChild(root);
|
||
conversation={root:root,orb:orb,stateEl:stateEl,userCaption:userCaption,assistantCaption:assistantCaption,muteBtn:muteBtn,exitBtn:exitBtn,langBtn:lang.btn,langMenu:lang.menu,langItems:lang.items,muted:false};
|
||
reflectLanguageSelection();
|
||
syncConversationOverlay(state);
|
||
if(root.focus) root.focus();
|
||
}catch(_){
|
||
// A partially built overlay is never attached: document.body.appendChild
|
||
// is the last DOM mutation above, so a stub DOM simply keeps the
|
||
// compact voice bar.
|
||
conversation=null;
|
||
}
|
||
}
|
||
|
||
function removeConversationOverlay(){
|
||
clearAwaitingAffordance();
|
||
const overlay=conversation;
|
||
conversation=null;
|
||
if(!overlay) return;
|
||
try{
|
||
if(overlay.root.removeEventListener) overlay.root.removeEventListener('keydown',conversationKeydown);
|
||
if(overlay.root.parentNode) overlay.root.parentNode.removeChild(overlay.root);
|
||
}catch(_){ }
|
||
// The session's tracks are always left enabled on the way out; a real
|
||
// deactivation stops them entirely via releaseMicrophone().
|
||
try{
|
||
const tracks=stream&&stream.getAudioTracks?stream.getAudioTracks():[];
|
||
tracks.forEach(function(track){track.enabled=true;});
|
||
}catch(_){ }
|
||
}
|
||
|
||
function clearErrorTimer(){
|
||
if(errorTimer){window.clearTimeout(errorTimer);errorTimer=null;}
|
||
}
|
||
|
||
function errorMessage(error,fallback){
|
||
return String((error&&error.message)||fallback||'Voice unavailable').trim();
|
||
}
|
||
|
||
function createAbortController(){
|
||
const Controller=window.AbortController||(typeof AbortController!=='undefined'?AbortController:null);
|
||
return Controller?new Controller():{signal:undefined,abort:function(){ }};
|
||
}
|
||
|
||
function cancelledError(message){
|
||
const error=new Error(message||'Voice turn cancelled');
|
||
error.name='AbortError';
|
||
return error;
|
||
}
|
||
|
||
function cancelSpeechTurn(){
|
||
if(!speechTurn) return;
|
||
cancelSpeakingIdleFallback(speechTurn);
|
||
speechTurn.cancelled=true;
|
||
speechTurn.final=true;
|
||
while(speechTurn.waiters.length) speechTurn.waiters.shift()(null);
|
||
speechTurn=null;
|
||
}
|
||
|
||
function cancelStreamingStt(){
|
||
if(!streamingStt) return;
|
||
streamingStt.cancel();
|
||
streamingStt=null;
|
||
}
|
||
|
||
function cancelVoicePreflight(turnId){
|
||
const preflight=voicePreflight;
|
||
if(!preflight||(turnId&&preflight.turnId!==turnId)) return;
|
||
voicePreflight=null;
|
||
if(preflight.timer){window.clearTimeout(preflight.timer);preflight.timer=null;}
|
||
if(preflight.deadline){window.clearTimeout(preflight.deadline);preflight.deadline=null;}
|
||
if(preflight.controller){preflight.controller.abort();preflight.controller=null;}
|
||
}
|
||
|
||
function scheduleVoicePreflight(turnId,revision,transcript){
|
||
const capability=streamingCapability.preflight;
|
||
const stable=String(transcript||'').replace(/\s+/g,' ').trim();
|
||
if(!capability||!voiceTabNonce||!active||captureTurnId!==turnId||state!=='listening') return;
|
||
if(!Number.isInteger(revision)||revision<1||stable.length<12||stable.length>512) return;
|
||
if(voicePreflight&&voicePreflight.turnId===turnId&&voicePreflight.transcript===stable) return;
|
||
cancelVoicePreflight();
|
||
const preflight={turnId:turnId,revision:revision,transcript:stable,timer:null,deadline:null,controller:null};
|
||
voicePreflight=preflight;
|
||
preflight.timer=window.setTimeout(async function(){
|
||
preflight.timer=null;
|
||
if(voicePreflight!==preflight||!active||captureTurnId!==turnId||state!=='listening') return;
|
||
const controller=createAbortController();
|
||
preflight.controller=controller;
|
||
preflight.deadline=window.setTimeout(function(){controller.abort();},VOICE_PREFLIGHT_DEADLINE_MS);
|
||
try{
|
||
const response=await fetch(VOICE_PREFLIGHT_URL,{
|
||
method:'POST',
|
||
credentials:'same-origin',
|
||
cache:'no-store',
|
||
headers:{'Content-Type':'application/json'},
|
||
body:JSON.stringify({turn_id:turnId,revision:revision,transcript:stable}),
|
||
signal:controller.signal,
|
||
});
|
||
if(!response.ok) return;
|
||
const advisory=await response.json();
|
||
const tier=advisory&&advisory.tier;
|
||
if(
|
||
voicePreflight!==preflight||!active||captureTurnId!==turnId||state!=='listening'||
|
||
advisory.turn_id!==turnId||advisory.revision!==revision||advisory.advisory!==true||
|
||
VOICE_PREFLIGHT_TIERS.indexOf(tier)<0||advisory.target!=='atlas/auto/'+tier
|
||
) return;
|
||
// This is intentionally feedback-only. The completed transcript still
|
||
// takes the normal Switchyard path and remains the sole routing input.
|
||
const preview=stable.length>60?stable.slice(0,57)+'…':stable;
|
||
label.textContent='Listening · '+preview+' · '+tier;
|
||
}catch(_){
|
||
// A partial can be superseded at any time; advisory failure is silent.
|
||
}finally{
|
||
if(preflight.deadline){window.clearTimeout(preflight.deadline);preflight.deadline=null;}
|
||
preflight.controller=null;
|
||
}
|
||
},VOICE_PREFLIGHT_DEBOUNCE_MS);
|
||
}
|
||
|
||
function cancelThinkingCues(){
|
||
const cue=thinkingCue;
|
||
thinkingCue=null;
|
||
if(!cue) return;
|
||
cue.cancelled=true;
|
||
if(cue.timer){window.clearTimeout(cue.timer);cue.timer=null;}
|
||
if(cue.controller){cue.controller.abort();cue.controller=null;}
|
||
if(cue.audioWake){const wake=cue.audioWake;cue.audioWake=null;wake();}
|
||
if(cue.node){try{cue.node.port.postMessage({type:'cancel'});cue.node.disconnect();}catch(_){ }cue.node=null;}
|
||
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
||
cue.gain=null;
|
||
indicator.classList.remove('is-playing');
|
||
indicator.classList.remove('is-ducked');
|
||
}
|
||
|
||
function clearBargeCancellation(){
|
||
const cancellation=bargeCancelPromise;
|
||
bargeCancelPromise=null;
|
||
if(cancellation&&cancellation.controller) cancellation.controller.abort();
|
||
}
|
||
|
||
function microphoneConstraints(){
|
||
const constraints={echoCancellation:true,noiseSuppression:true,autoGainControl:true};
|
||
const supported=navigator.mediaDevices.getSupportedConstraints?navigator.mediaDevices.getSupportedConstraints():{};
|
||
if(supported.voiceIsolation) constraints.voiceIsolation=true;
|
||
return constraints;
|
||
}
|
||
|
||
function acquireMicrophone(){
|
||
return navigator.mediaDevices.getUserMedia({audio:microphoneConstraints()});
|
||
}
|
||
|
||
function captureAecIsUsable(capture){
|
||
try{
|
||
const tracks=capture&&capture.getAudioTracks?capture.getAudioTracks():[];
|
||
const settings=tracks.length&&tracks[0].getSettings?tracks[0].getSettings():{};
|
||
return settings.echoCancellation!==false;
|
||
}catch(_){
|
||
return true;
|
||
}
|
||
}
|
||
|
||
window._atlasCaptureAecIsUsable=captureAecIsUsable;
|
||
|
||
function setPlaybackDucked(ducked){
|
||
const session=playbackSession;
|
||
if(session&&session.gain&&session.context){
|
||
const gain=ducked?0.16:1;
|
||
try{session.gain.gain.setTargetAtTime(gain,session.context.currentTime,0.018);}catch(_){session.gain.gain.value=gain;}
|
||
}
|
||
if(currentAudio) currentAudio.volume=ducked?0.16:1;
|
||
const cue=thinkingCue;
|
||
if(cue&&cue.gain&&cue.context){
|
||
const gain=ducked?0.16:1;
|
||
try{cue.gain.gain.setTargetAtTime(gain,cue.context.currentTime,0.018);}catch(_){cue.gain.gain.value=gain;}
|
||
}
|
||
if(ducked) indicator.classList.add('is-ducked'); else indicator.classList.remove('is-ducked');
|
||
}
|
||
|
||
function playbackAudible(){
|
||
// True only while audio is actually sounding. A streaming PCM session
|
||
// keeps its worklet node alive across sentence chunks and between the
|
||
// speech segments of one turn; the buffered frames the worklet reports
|
||
// decide whether it is audible right now.
|
||
if(currentAudio) return true;
|
||
if(thinkingCue&&thinkingCue.node) return true;
|
||
const session=playbackSession;
|
||
if(session&&session.node) return session.bufferedFrames>0&&!session.playbackEnded;
|
||
return indicator.classList.contains('is-playing');
|
||
}
|
||
|
||
function disposeBargeResources(monitor,keepCapture){
|
||
if(!monitor) return;
|
||
monitor.cancelled=true;
|
||
if(monitor.timer){window.clearInterval(monitor.timer);monitor.timer=null;}
|
||
try{if(monitor.node) monitor.node.disconnect();}catch(_){ }
|
||
try{if(monitor.source) monitor.source.disconnect();}catch(_){ }
|
||
try{if(monitor.silentGain) monitor.silentGain.disconnect();}catch(_){ }
|
||
if(!keepCapture){
|
||
if(monitor.stream) monitor.stream.getTracks().forEach(function(track){track.stop();});
|
||
if(monitor.context){try{monitor.context.close();}catch(_){ }}
|
||
}
|
||
if(!keepCapture) setPlaybackDucked(false);
|
||
}
|
||
|
||
function stopBargeMonitor(){
|
||
const monitor=bargeMonitor;
|
||
bargeMonitor=null;
|
||
disposeBargeResources(monitor,false);
|
||
}
|
||
|
||
function showUnavailable(message){
|
||
generation+=1;
|
||
active=false;
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
suppressAutoRead=false;
|
||
pendingStitch=null;
|
||
lastSentTranscript=null;
|
||
clearBargeCancellation();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
releaseMicrophone();
|
||
stopPlayback();
|
||
removeConversationOverlay();
|
||
modeBtn.classList.remove('active');
|
||
setState('error',message);
|
||
clearErrorTimer();
|
||
errorTimer=window.setTimeout(function(){
|
||
errorTimer=null;
|
||
if(!active&&state==='error'&&indicator.classList.contains('error')) setState('idle');
|
||
},ERROR_VISIBLE_MS);
|
||
}
|
||
|
||
function stopCapture(){
|
||
// Tear down the per-utterance capture pipeline (recorder, VAD, streaming
|
||
// STT, WebAudio graph) but deliberately KEEP the microphone stream and its
|
||
// AudioContext, so the next capture turn starts instantly and speech
|
||
// during transcribing/thinking is never lost. Only releaseMicrophone()
|
||
// actually ends the session's single getUserMedia lease.
|
||
captureGeneration+=1;
|
||
captureActive=false;
|
||
cancelVoicePreflight(captureTurnId);
|
||
stopBargeMonitor();
|
||
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
||
if(recorder&&recorder.state!=='inactive'){
|
||
try{recorder.stop();}catch(_){ }
|
||
}
|
||
recorder=null;
|
||
if(captureNode){try{captureNode.disconnect();}catch(_){ }}
|
||
captureNode=null;
|
||
if(captureGraph){
|
||
// The AudioContext persists across turns, so every per-turn node must be
|
||
// detached here or the shared graph would grow with each utterance.
|
||
try{if(captureGraph.mediaSource) captureGraph.mediaSource.disconnect();}catch(_){ }
|
||
try{if(captureGraph.highpass) captureGraph.highpass.disconnect();}catch(_){ }
|
||
try{if(captureGraph.analyser) captureGraph.analyser.disconnect();}catch(_){ }
|
||
try{if(captureGraph.silentGain) captureGraph.silentGain.disconnect();}catch(_){ }
|
||
captureGraph=null;
|
||
}
|
||
cancelStreamingStt();
|
||
}
|
||
|
||
function releaseMicrophone(){
|
||
// Full microphone teardown. Used only by deactivate(), showUnavailable()
|
||
// and fatal capture errors; every other path retains the hot microphone.
|
||
stopCapture();
|
||
if(stream){stream.getTracks().forEach(function(track){track.stop();});stream=null;}
|
||
if(audioContext){try{audioContext.close();}catch(_){ }audioContext=null;}
|
||
}
|
||
|
||
function stopPlayback(){
|
||
if(currentAudio){
|
||
try{currentAudio.pause();currentAudio.currentTime=0;}catch(_){ }
|
||
currentAudio=null;
|
||
}
|
||
if(playbackSession){
|
||
playbackSession.cancelled=true;
|
||
if(playbackSession.blobWake){playbackSession.blobWake();playbackSession.blobWake=null;}
|
||
playbackSession.controllers.forEach(function(controller){controller.abort();});
|
||
if(playbackSession.lowWaterWake){
|
||
playbackSession.lowWaterWake();
|
||
playbackSession.lowWaterWake=null;
|
||
}
|
||
if(playbackSession.drainWake){playbackSession.drainWake();playbackSession.drainWake=null;}
|
||
if(playbackSession.node){
|
||
try{playbackSession.node.port.postMessage({type:'cancel'});playbackSession.node.disconnect();}catch(_){ }
|
||
}
|
||
if(playbackSession.context){try{playbackSession.context.close();}catch(_){ }}
|
||
playbackSession=null;
|
||
}
|
||
indicator.classList.remove('is-playing');
|
||
indicator.classList.remove('is-ducked');
|
||
}
|
||
|
||
function stopResponseObserver(){
|
||
if(responsePollTimer){clearInterval(responsePollTimer);responsePollTimer=null;}
|
||
}
|
||
|
||
function deactivate(showMessage){
|
||
generation+=1;
|
||
active=false;
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
clearErrorTimer();
|
||
removeConversationOverlay();
|
||
clearSttLanguage();
|
||
sessionLanguage='';
|
||
forcedLanguage='';
|
||
finalizeAttempts=0;
|
||
suppressAutoRead=false;
|
||
pendingStitch=null;
|
||
lastSentTranscript=null;
|
||
clearBargeCancellation();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
releaseMicrophone();
|
||
stopPlayback();
|
||
modeBtn.classList.remove('active');
|
||
setState('idle');
|
||
if(showMessage) toast('Hands-free voice mode off');
|
||
}
|
||
|
||
function restartSoon(token,delay){
|
||
window.setTimeout(function(){
|
||
if(!active||token!==generation) return;
|
||
// Continuous capture: while the microphone pipeline is already running,
|
||
// finishing a response only needs the display returned to Listening.
|
||
// Rebuilding capture here would drop speech already being collected.
|
||
if(captureActive){setState('listening');return;}
|
||
startListening(token);
|
||
},delay||500);
|
||
}
|
||
|
||
function resyncCapture(token,statusLabel){
|
||
// Full capture-turn resync: cancel any streaming STT session, drop the
|
||
// lookback/pending buffers and start a fresh capture turn on the hot
|
||
// microphone. Used after an errored turn and by barge-in edge cases
|
||
// where the previous stream state is no longer trustworthy.
|
||
if(!active||token!==generation) return;
|
||
stopCapture();
|
||
startListening(token);
|
||
setState('listening',statusLabel);
|
||
}
|
||
|
||
function handleAssistantResponseError(token){
|
||
// An error/system envelope (cancellation notice, provider failure) is a
|
||
// transcript artifact, not a reply: never feed it to TTS or the reply
|
||
// caption, show a brief non-spoken state instead, and resynchronize
|
||
// capture so the next utterance starts a clean streaming turn.
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
finalizeAttempts=0;
|
||
clearSttLanguage();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
stopPlayback();
|
||
pendingStitch=null;
|
||
lastSentTranscript=null;
|
||
clearBargeCancellation();
|
||
setConversationAssistantCaption('');
|
||
resyncCapture(token,'Let’s try that again — listening');
|
||
}
|
||
|
||
function assistantRows(){
|
||
return document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]');
|
||
}
|
||
|
||
function assistantTurnOf(row){
|
||
// Resolve a matched node to its whole turn container so multi-segment
|
||
// turns (interim messages around tool calls) are read as one response.
|
||
if(row&&typeof row.closest==='function'){
|
||
const turn=row.closest('.msg-row[data-role="assistant"]');
|
||
if(turn) return turn;
|
||
}
|
||
return row;
|
||
}
|
||
|
||
function segmentIsHidden(segment){
|
||
if(!segment) return true;
|
||
if(segment.hidden===true) return true;
|
||
return !!(typeof segment.getAttribute==='function'&&segment.getAttribute('aria-hidden')==='true');
|
||
}
|
||
|
||
function segmentIsError(segment){
|
||
// Error/system envelopes: the renderer stamps data-error="1" on segments
|
||
// matching its error patterns, and provider errors and cancellation
|
||
// notices carry a .provider-error-details block inside the body.
|
||
if(!segment) return false;
|
||
if(segment.dataset&&segment.dataset.error==='1') return true;
|
||
return !!(typeof segment.querySelector==='function'&&segment.querySelector('.provider-error-details'));
|
||
}
|
||
|
||
// Ground-truth DOM contract (verified against the live build-24 bundle,
|
||
// ui.js renderMessages / messages.js ensureAssistantRow):
|
||
//
|
||
// <div class="msg-row assistant-turn" data-role="assistant">
|
||
// <div class="msg-role assistant">
|
||
// <div class="role-icon assistant">H</div> ← avatar letter
|
||
// <span class="msg-role-name">Hermes</span> ← author name
|
||
// </div>
|
||
// <div class="assistant-turn-blocks">
|
||
// <div class="tool-worklog-group">… "Processed 13s" …</div> ← worklog chip
|
||
// <div class="assistant-segment assistant-segment-worklog-source"
|
||
// hidden aria-hidden="true" data-raw-text="…">…</div> ← folded interim
|
||
// <div class="assistant-segment" data-raw-text="ANSWER">
|
||
// <div class="thinking-card">…</div> ← reasoning (optional)
|
||
// <div class="msg-body">ANSWER</div> ← the spoken answer
|
||
// </div>
|
||
// </div>
|
||
// </div>
|
||
//
|
||
// The settled renderer stamps the CLEAN answer text (no avatar, no author, no
|
||
// "Processed Ns" chip, no reasoning) onto every answer segment's data-raw-text
|
||
// — the exact field the app's own autoReadLastAssistant() reads (ui.js:~8891).
|
||
// Reasoning uses .thinking-card-body, tool activity uses .tool-call-group-body
|
||
// and the worklog "Processed Ns" chip lives in the worklog group beside the
|
||
// segments — so .msg-body is answer-only, and the concatenated data-raw-text of
|
||
// the visible answer segments is the whole reply. The prior round's fallback to
|
||
// turn.textContent scraped the avatar "H" + "Hermes" + "Processed 13s" whenever
|
||
// a settle-frame race left no readable answer segment; that fallback is gone.
|
||
const NON_ANSWER_SEGMENT_CLASSES=['assistant-segment-worklog-source','assistant-segment-anchor'];
|
||
// A .msg-body is the spoken answer ONLY when it is not nested inside reasoning,
|
||
// tool, worklog, run-status, error or role chrome. The live renderer uses
|
||
// distinct classes for each, but this guard keeps chrome out of the caption and
|
||
// TTS even if a future build nests a body differently.
|
||
const CHROME_CONTAINER_SELECTOR='.thinking-card,.agent-activity-thinking,.tool-call-group,.tool-worklog-group,.tool-group,.tool-card,.wl-reason,.process-wakeup-card,.provider-error-details,.msg-role,.assistant-run-status,.assistant-segment-worklog-source';
|
||
|
||
function segmentHasClass(segment,name){
|
||
if(!segment) return false;
|
||
if(segment.classList&&typeof segment.classList.contains==='function') return segment.classList.contains(name);
|
||
return typeof segment.className==='string'&&segment.className.split(/\s+/).indexOf(name)>=0;
|
||
}
|
||
|
||
function segmentIsAnswer(segment){
|
||
// A visible answer segment: not folded into the worklog, not an anchor
|
||
// scaffold, not hidden, not an error/system envelope.
|
||
if(!segment) return false;
|
||
if(segmentIsHidden(segment)) return false;
|
||
if(segmentIsError(segment)) return false;
|
||
for(let i=0;i<NON_ANSWER_SEGMENT_CLASSES.length;i+=1){
|
||
if(segmentHasClass(segment,NON_ANSWER_SEGMENT_CLASSES[i])) return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function answerBodiesText(segment){
|
||
// Concatenate ONLY answer .msg-body text, excluding any body that sits inside
|
||
// reasoning / tool / worklog / status chrome.
|
||
if(!segment||typeof segment.querySelectorAll!=='function') return '';
|
||
const bodies=segment.querySelectorAll('.msg-body');
|
||
if(!bodies||!bodies.length) return '';
|
||
const parts=[];
|
||
Array.prototype.forEach.call(bodies,function(body){
|
||
if(body.classList&&typeof body.classList.contains==='function'&&body.classList.contains('process-wakeup-body')) return;
|
||
if(typeof body.closest==='function'&&body.closest(CHROME_CONTAINER_SELECTOR)) return;
|
||
const value=typeof body.textContent==='string'?body.textContent:'';
|
||
if(value) parts.push(value);
|
||
});
|
||
return parts.join('\n');
|
||
}
|
||
|
||
function readSegmentBody(segment){
|
||
if(!segment) return '';
|
||
// Prefer the renderer-stamped clean answer text; while a turn is still
|
||
// streaming the live segment has no data-raw-text yet, so read its answer
|
||
// .msg-body directly. Never fall through to the segment's whole textContent.
|
||
if(segment.dataset&&typeof segment.dataset.rawText==='string'&&segment.dataset.rawText!=='') return segment.dataset.rawText;
|
||
return answerBodiesText(segment);
|
||
}
|
||
|
||
function readAssistantTurn(turn){
|
||
// {text, error}: the concatenated human-visible ANSWER text of every visible
|
||
// answer segment of the turn — never the avatar letter, the "Hermes" author
|
||
// name, a "Processed Ns" worklog chip, reasoning or tool chrome — plus whether
|
||
// any segment is an error/system envelope. Returns '' (NEVER the row's
|
||
// textContent) when the turn has no readable answer segment, so a tool-only or
|
||
// not-yet-answered settle frame can never leak chrome into the caption or the
|
||
// one-ahead TTS synthesizer.
|
||
if(!turn) return {text:'',error:false};
|
||
let error=false;
|
||
const parts=[];
|
||
const segments=(typeof turn.querySelectorAll==='function')?turn.querySelectorAll('.assistant-segment'):null;
|
||
if(segments&&segments.length){
|
||
Array.prototype.forEach.call(segments,function(segment){
|
||
if(segmentIsError(segment)){error=true;return;}
|
||
if(!segmentIsAnswer(segment)) return;
|
||
const value=readSegmentBody(segment);
|
||
if(value&&value.trim()) parts.push(value.trim());
|
||
});
|
||
}else if(segmentIsError(turn)){
|
||
error=true;
|
||
}else if(segmentIsAnswer(turn)){
|
||
// The matched node was itself a lone answer segment (a leaf's
|
||
// querySelectorAll returns nothing): read its own stamped answer text.
|
||
const value=readSegmentBody(turn);
|
||
if(value&&value.trim()) parts.push(value.trim());
|
||
}
|
||
// A stray error-stamped segment (a recovered tool error, or a
|
||
// cancellation notice from an earlier interim) must not discard a real
|
||
// answer: only surface the error state when the turn produced no
|
||
// spoken answer at all.
|
||
return {text:cleanForSpeech(parts.join('\n\n')),error:error&&parts.length===0};
|
||
}
|
||
|
||
function rememberAssistantBaseline(){
|
||
const rows=assistantRows();
|
||
const turn=rows.length?assistantTurnOf(rows[rows.length-1]):null;
|
||
assistantBaseline={row:turn,text:readAssistantTurn(turn).text,count:rows.length};
|
||
}
|
||
|
||
async function settleBargeCancellation(token){
|
||
const cancellation=bargeCancelPromise;
|
||
if(!cancellation) return true;
|
||
if(!cancellation.streamId){
|
||
// Nothing was actually in flight to cancel: a stale busy flag left by
|
||
// an errored turn must never hold the send in a dead settle wait.
|
||
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
|
||
return true;
|
||
}
|
||
try{await cancellation.promise;}catch(_){ }
|
||
const deadline=Date.now()+10000;
|
||
while(active&&token===generation&&Date.now()<deadline){
|
||
const sessionId=(typeof S!=='undefined'&&S.session)?S.session.session_id:'';
|
||
const activeStreamId=typeof S!=='undefined'?String(S.activeStreamId||''):'';
|
||
const busy=typeof S!=='undefined'&&!!S.busy;
|
||
if(sessionId!==cancellation.sessionId) break;
|
||
if(!busy&&(!cancellation.streamId||activeStreamId!==cancellation.streamId)){
|
||
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
|
||
return true;
|
||
}
|
||
await new Promise(function(resolve){window.setTimeout(resolve,50);});
|
||
}
|
||
if(bargeCancelPromise===cancellation) bargeCancelPromise=null;
|
||
return false;
|
||
}
|
||
|
||
function recordPendingStitch(){
|
||
// Called at the moment a model turn is cancelled by user speech. The
|
||
// interrupted send left the user mid-thought regardless of how much of an
|
||
// answer had already streamed — responses start streaming almost
|
||
// immediately, so gating on visible output made stitching almost never
|
||
// fire. The next finalized utterance inside STITCH_WINDOW_MS is therefore
|
||
// always sent together with the interrupted transcript as one message.
|
||
// Normal completion, deactivation and window expiry still clear it.
|
||
const sent=lastSentTranscript;
|
||
lastSentTranscript=null;
|
||
if(!sent||!sent.text) return;
|
||
// The playback queue knows exactly where the user stopped listening: the
|
||
// chunk being spoken (or the last fully spoken one) becomes the cut
|
||
// marker appended to the stitched follow-up message.
|
||
const turn=speechTurn;
|
||
const cut=turn?String(turn.speakingChunk||turn.lastSpokenChunk||''):'';
|
||
pendingStitch={text:sent.text,at:Date.now(),cut:cut};
|
||
}
|
||
|
||
function voiceCutMarker(cut){
|
||
// Exactly one compact machine-readable line, appended to a stitched
|
||
// interruption, telling the model where its spoken reply was cut off so
|
||
// it can resume naturally instead of re-answering from the top.
|
||
const tail=String(cut||'').replace(/\s+/g,' ').replace(/"/g,"'").trim();
|
||
if(!tail) return '';
|
||
const bounded=tail.length>120?tail.slice(tail.length-120):tail;
|
||
return '\n[voice interruption: you were cut off after "'+bounded+'"]';
|
||
}
|
||
|
||
async function sendTranscript(transcript,token,language,turnId){
|
||
if(!active||token!==generation) return;
|
||
cancelVoicePreflight(turnId||captureTurnId);
|
||
const text=String(transcript||'').trim();
|
||
if(!text){clearSttLanguage();restartSoon(token,350);return;}
|
||
if(!bargeCancelPromise&&typeof S!=='undefined'&&(S.busy||S.activeStreamId)){
|
||
// A new utterance finished while the previous model turn was still in
|
||
// flight (continuous capture makes this a normal interruption): remember
|
||
// the interrupted transcript for stitching, then cancel the stale turn.
|
||
recordPendingStitch();
|
||
suppressAutoRead=true;
|
||
bargeCancelPromise=cancelActiveModelTurn();
|
||
}
|
||
// A turn whose response was barge-cancelled before any assistant output is
|
||
// restated as one stitched message, so the model answers the full thought.
|
||
const stitch=(pendingStitch&&pendingStitch.text&&(Date.now()-pendingStitch.at)<=STITCH_WINDOW_MS)?pendingStitch:null;
|
||
pendingStitch=null;
|
||
composer.value=stitch?stitch.text+' '+text+voiceCutMarker(stitch.cut):text;
|
||
if(typeof window.autoResize==='function') window.autoResize();
|
||
setConversationUserCaption(composer.value);
|
||
setConversationAssistantCaption('');
|
||
setState('thinking');
|
||
// The live VAD/STT capture is the barge-in detector while it runs; the
|
||
// energy-only monitor remains only as a fallback when capture is down.
|
||
if(!captureActive) startBargeMonitor(token);
|
||
const cancellationSettled=await settleBargeCancellation(token);
|
||
if(!active||token!==generation) return;
|
||
if(!cancellationSettled){
|
||
pendingStitch=stitch;
|
||
toast('The previous response did not stop. Please repeat your interruption.');
|
||
// The stream state is now suspect: rebuild the capture turn instead of
|
||
// resuming a session whose server-side epoch may be stale.
|
||
resyncCapture(token);
|
||
return;
|
||
}
|
||
thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
||
thinkingTurnId=turnId||captureTurnId||String(token)+'-'+String(++turnSequence);
|
||
rememberAssistantBaseline();
|
||
rememberSttLanguage(language,token);
|
||
// The user's detected speech language is the sticky hands-free session
|
||
// language: it biases the next streaming STT session and localizes the
|
||
// thinking cues until the user audibly switches again.
|
||
if(language){
|
||
const switched=language!==sessionLanguage;
|
||
sessionLanguage=language;
|
||
// Continuous capture already opened the next session (with the
|
||
// previous bias) before this turn's detection arrived: restart it with
|
||
// the switched language, but only while it has heard nothing yet.
|
||
if(switched&&captureActive&&streamingStt&&typeof streamingStt.latestPartial==='function'&&!streamingStt.latestPartial()){
|
||
startListening(generation,{preserveDisplay:true});
|
||
}
|
||
}
|
||
if(typeof window.send==='function'){
|
||
lastSentTranscript={text:composer.value,token:token};
|
||
window.send();
|
||
suppressAutoRead=false;
|
||
startResponseObserver(token);
|
||
scheduleThinkingCues(token,language||sessionLanguage,thinkingTurnId);
|
||
}
|
||
}
|
||
|
||
function audioExtension(mimeType){
|
||
const normalized=String(mimeType||'').toLowerCase();
|
||
if(normalized.indexOf('wav')>=0) return 'wav';
|
||
if(normalized.indexOf('ogg')>=0) return 'ogg';
|
||
if(normalized.indexOf('mp4')>=0) return 'mp4';
|
||
return 'webm';
|
||
}
|
||
|
||
async function transcribe(blob,token,turnId){
|
||
if(!active||token!==generation) return;
|
||
setState('transcribing');
|
||
const ext=audioExtension(blob.type);
|
||
const form=new FormData();
|
||
form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
|
||
try{
|
||
const response=await fetch('/api/transcribe',{method:'POST',body:form});
|
||
const payload=await response.json().catch(function(){return {};});
|
||
if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status));
|
||
sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language),turnId);
|
||
}catch(error){
|
||
if(!active||token!==generation) return;
|
||
const message=errorMessage(error,'Private Whisper is unavailable');
|
||
showUnavailable(message);
|
||
toast(message);
|
||
// If the browser supplies its own recognizer, hand control back to the
|
||
// upstream voice implementation until the Jetson becomes healthy again.
|
||
if(window.SpeechRecognition||window.webkitSpeechRecognition){
|
||
modeBtn.removeEventListener('click',onVoiceClick,true);
|
||
window.setTimeout(function(){modeBtn.click();},50);
|
||
}
|
||
}
|
||
}
|
||
|
||
function websocketUrl(path){
|
||
const protocol=window.location&&window.location.protocol==='https:'?'wss:':'ws:';
|
||
return protocol+'//'+window.location.host+path;
|
||
}
|
||
|
||
function encodePcm16(samples){
|
||
const bytes=new ArrayBuffer(samples.length*2);
|
||
const view=new DataView(bytes);
|
||
for(let index=0;index<samples.length;index+=1){
|
||
const value=Math.max(-1,Math.min(1,samples[index]));
|
||
view.setInt16(index*2,value<0?value*32768:value*32767,true);
|
||
}
|
||
return bytes;
|
||
}
|
||
|
||
function pcm16WavBlob(buffers,sampleRate){
|
||
const chunks=Array.isArray(buffers)?buffers:[];
|
||
const pcmBytes=chunks.reduce(function(total,chunk){return total+(chunk?chunk.byteLength:0);},0);
|
||
const wav=new ArrayBuffer(44+pcmBytes);
|
||
const view=new DataView(wav);
|
||
const write=function(offset,value){for(let index=0;index<value.length;index+=1)view.setUint8(offset+index,value.charCodeAt(index));};
|
||
write(0,'RIFF');
|
||
view.setUint32(4,36+pcmBytes,true);
|
||
write(8,'WAVE');
|
||
write(12,'fmt ');
|
||
view.setUint32(16,16,true);
|
||
view.setUint16(20,1,true);
|
||
view.setUint16(22,1,true);
|
||
view.setUint32(24,sampleRate,true);
|
||
view.setUint32(28,sampleRate*2,true);
|
||
view.setUint16(32,2,true);
|
||
view.setUint16(34,16,true);
|
||
write(36,'data');
|
||
view.setUint32(40,pcmBytes,true);
|
||
const output=new Uint8Array(wav,44);
|
||
let offset=0;
|
||
chunks.forEach(function(chunk){const bytes=new Uint8Array(chunk);output.set(bytes,offset);offset+=bytes.length;});
|
||
return new Blob([wav],{type:'audio/wav'});
|
||
}
|
||
|
||
window._atlasPcm16WavBlob=pcm16WavBlob;
|
||
|
||
function createResampler(sourceRate,targetRate){
|
||
let tail=null;
|
||
let position=0;
|
||
const ratio=sourceRate/targetRate;
|
||
return function(samples){
|
||
const input=new Float32Array(samples.length+(tail===null?0:1));
|
||
let offset=0;
|
||
if(tail!==null){input[0]=tail;offset=1;}
|
||
input.set(samples,offset);
|
||
if(input.length<2){tail=input.length?input[0]:tail;return new Float32Array(0);}
|
||
const output=[];
|
||
while(position<input.length-1){
|
||
const left=Math.floor(position);
|
||
const fraction=position-left;
|
||
output.push(input[left]+((input[left+1]-input[left])*fraction));
|
||
position+=ratio;
|
||
}
|
||
position-=input.length-1;
|
||
tail=input[input.length-1];
|
||
return new Float32Array(output);
|
||
};
|
||
}
|
||
|
||
function createStreamingSttSession(turnId,context){
|
||
if(!streamingCapability.stt||!window.WebSocket||!window.location) return null;
|
||
let socket;
|
||
let settled=false;
|
||
let committed=false;
|
||
let speculative=false;
|
||
let workletReady=false;
|
||
let partialRevision=-1;
|
||
let lastPreflightText='';
|
||
let lastPartialText='';
|
||
let queue=[];
|
||
let queuedBytes=0;
|
||
let flushTimer=null;
|
||
let archive=[];
|
||
let archiveBytes=0;
|
||
let cancelled=false;
|
||
let resolveFinal;
|
||
let rejectFinal;
|
||
const finalPromise=new Promise(function(resolve,reject){resolveFinal=resolve;rejectFinal=reject;});
|
||
// A transport can fail before recorder.onstop awaits it. Keep the rejection
|
||
// observed without changing what the later await receives.
|
||
finalPromise.catch(function(){ });
|
||
const resample=createResampler(context.sampleRate,16000);
|
||
|
||
function clearFlushTimer(){
|
||
if(flushTimer){window.clearTimeout(flushTimer);flushTimer=null;}
|
||
}
|
||
function clearQueue(){queue=[];queuedBytes=0;clearFlushTimer();}
|
||
function clearArchive(){archive=[];archiveBytes=0;}
|
||
function archivePcm(bytes){
|
||
if(cancelled||!bytes||!bytes.byteLength||archiveBytes>=STT_MAX_ARCHIVE_BYTES) return;
|
||
const retained=Math.min(bytes.byteLength,STT_MAX_ARCHIVE_BYTES-archiveBytes);
|
||
const even=retained-(retained%2);
|
||
if(!even) return;
|
||
archive.push(bytes.slice(0,even));
|
||
archiveBytes+=even;
|
||
}
|
||
function reject(error){
|
||
if(settled) return;
|
||
settled=true;
|
||
cancelVoicePreflight(turnId);
|
||
clearQueue();
|
||
rejectFinal(error instanceof Error?error:new Error(String(error||'Streaming transcription failed')));
|
||
}
|
||
function sendJson(payload){
|
||
if(socket&&socket.readyState===1){socket.send(JSON.stringify(payload));return true;}
|
||
return false;
|
||
}
|
||
function scheduleFlush(){
|
||
if(flushTimer||settled||!queue.length||!socket||socket.readyState!==1) return;
|
||
flushTimer=window.setTimeout(function(){flushTimer=null;flush();},20);
|
||
}
|
||
function flush(){
|
||
clearFlushTimer();
|
||
if(settled){clearQueue();return;}
|
||
if(!socket||socket.readyState!==1) return;
|
||
while(queue.length&&socket.bufferedAmount<524288){
|
||
const bytes=queue.shift();
|
||
queuedBytes=Math.max(0,queuedBytes-bytes.byteLength);
|
||
socket.send(bytes);
|
||
}
|
||
if(queue.length) scheduleFlush();
|
||
}
|
||
try{
|
||
const csrf=String((window.__HERMES_CONFIG__&&window.__HERMES_CONFIG__.csrfToken)||'');
|
||
const protocols=['hermes-voice-v1'];
|
||
if(csrf) protocols.push('hermes-csrf.'+csrf);
|
||
socket=new WebSocket(websocketUrl(STT_STREAM_PATH),protocols);
|
||
socket.binaryType='arraybuffer';
|
||
}catch(error){
|
||
reject(error);
|
||
return null;
|
||
}
|
||
socket.onopen=function(){
|
||
// Bias recognition toward the session's sticky language the moment the
|
||
// user has audibly switched; 'auto' remains the cold-start default.
|
||
sendJson({type:'start',turn_id:turnId,format:'pcm_s16le',sample_rate:16000,language:forcedLanguage||sessionLanguage||'auto'});
|
||
flush();
|
||
};
|
||
socket.onmessage=function(event){
|
||
if(typeof event.data!=='string') return;
|
||
let payload;
|
||
try{payload=JSON.parse(event.data);}catch(_){return;}
|
||
if(payload.turn_id!==turnId) return;
|
||
if(payload.type==='partial'&&payload.rolling===true){
|
||
const revision=Number(payload.revision);
|
||
if(Number.isFinite(revision)&&revision>partialRevision){
|
||
partialRevision=revision;
|
||
const stable=String(payload.stable_transcript||'').trim();
|
||
const provisional=String(payload.transcript||'').trim();
|
||
const visible=stable||provisional;
|
||
// The newest rolling partial also feeds the dynamic endpoint below,
|
||
// so it is tracked regardless of the display-only gating that keeps
|
||
// the label and captions quiet outside the listening state.
|
||
if(visible) lastPartialText=visible;
|
||
if(visible&&active&&captureTurnId===turnId&&state==='listening'){
|
||
const preview=visible.length>72?visible.slice(0,69)+'…':visible;
|
||
label.textContent='Listening · '+preview+(stable?'':' · provisional');
|
||
setConversationUserCaption(visible);
|
||
if(stable!==lastPreflightText){
|
||
if(stable){
|
||
lastPreflightText=stable;
|
||
scheduleVoicePreflight(turnId,revision,stable);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}else if(payload.type==='final'&&!settled){
|
||
settled=true;
|
||
cancelVoicePreflight(turnId);
|
||
clearQueue();
|
||
clearArchive();
|
||
resolveFinal({transcript:payload.transcript||'',language:normalizeSttLanguage(payload.language)});
|
||
}else if(payload.type==='error'){
|
||
reject(new Error(payload.error||'Streaming transcription failed'));
|
||
}
|
||
};
|
||
socket.onerror=function(){reject(new Error('Streaming transcription connection failed'));};
|
||
socket.onclose=function(){if(!settled) reject(new Error('Streaming transcription closed before final result'));};
|
||
|
||
return {
|
||
finalPromise:finalPromise,
|
||
latestPartial:function(){return lastPartialText;},
|
||
setWorkletReady:function(value){workletReady=value;},
|
||
push:function(samples){
|
||
if(cancelled||committed||!workletReady) return;
|
||
const pcm=resample(samples);
|
||
if(!pcm.length) return;
|
||
const bytes=encodePcm16(pcm);
|
||
archivePcm(bytes);
|
||
if(settled) return;
|
||
if(queuedBytes+bytes.byteLength>STT_MAX_QUEUED_BYTES){
|
||
reject(new Error('Streaming transcription backpressure limit exceeded'));
|
||
if(socket&&socket.readyState<2) socket.close(1008,'client backpressure');
|
||
return;
|
||
}
|
||
queue.push(bytes);
|
||
queuedBytes+=bytes.byteLength;
|
||
flush();
|
||
},
|
||
takeFallbackBlob:function(){
|
||
if(!archiveBytes) return null;
|
||
const blob=pcm16WavBlob(archive,16000);
|
||
clearArchive();
|
||
return blob;
|
||
},
|
||
speculate:function(){
|
||
if(settled||committed||speculative) return;
|
||
speculative=true;
|
||
sendJson({type:'speculate',turn_id:turnId});
|
||
},
|
||
resume:function(){
|
||
// Deliberately not gated on a prior speculate() from this client: the
|
||
// server also freezes an EOS snapshot on its own silence detector, so
|
||
// resume must always reach the wire when speech restarts. The server
|
||
// treats a redundant resume as a harmless epoch bump.
|
||
if(settled||committed) return;
|
||
speculative=false;
|
||
sendJson({type:'resume',turn_id:turnId});
|
||
},
|
||
commit:function(){
|
||
if(settled||committed) return finalPromise;
|
||
committed=true;
|
||
flush();
|
||
const commitDeadline=Date.now()+30000;
|
||
const sendCommit=function(){
|
||
if(settled) return;
|
||
if(!socket||socket.readyState>1||Date.now()>=commitDeadline){
|
||
reject(new Error('Streaming transcription closed before commit'));
|
||
return;
|
||
}
|
||
if(socket.readyState!==1||queue.length){flush();window.setTimeout(sendCommit,20);return;}
|
||
if(!sendJson({type:'commit',turn_id:turnId})) reject(new Error('Streaming transcription commit failed'));
|
||
};
|
||
sendCommit();
|
||
return finalPromise;
|
||
},
|
||
cancel:function(){
|
||
cancelled=true;
|
||
cancelVoicePreflight(turnId);
|
||
clearQueue();
|
||
clearArchive();
|
||
if(!settled){sendJson({type:'cancel',turn_id:turnId});reject(new Error('Streaming transcription cancelled'));}
|
||
if(socket&&socket.readyState<2) socket.close(1000,'cancelled');
|
||
},
|
||
};
|
||
}
|
||
|
||
async function installCaptureWorklet(context,source,session){
|
||
if(!session) return false;
|
||
if(!context.audioWorklet||!window.AudioWorkletNode){
|
||
session.cancel();
|
||
if(streamingStt===session) streamingStt=null;
|
||
return false;
|
||
}
|
||
try{
|
||
await context.audioWorklet.addModule(WORKLET_URL);
|
||
if(!streamingStt||streamingStt!==session) return false;
|
||
const node=new AudioWorkletNode(context,'atlas-pcm-capture');
|
||
const silentGain=context.createGain();
|
||
silentGain.gain.value=0;
|
||
let flushResolve=null;
|
||
node.port.onmessage=function(event){
|
||
if(!event.data) return;
|
||
if(event.data.type==='flushed'&&flushResolve){flushResolve();flushResolve=null;return;}
|
||
if(!streamingStt||streamingStt!==session||event.data.type!=='pcm') return;
|
||
session.push(new Float32Array(event.data.samples));
|
||
};
|
||
node._atlasFlush=function(){
|
||
return new Promise(function(resolve){flushResolve=resolve;node.port.postMessage({type:'flush'});});
|
||
};
|
||
source.connect(node);
|
||
node.connect(silentGain);
|
||
silentGain.connect(context.destination);
|
||
captureNode=node;
|
||
if(captureGraph) captureGraph.silentGain=silentGain;
|
||
session.setWorkletReady(true);
|
||
return true;
|
||
}catch(_){
|
||
session.cancel();
|
||
if(streamingStt===session) streamingStt=null;
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function cancelActiveModelTurn(){
|
||
const sessionId=(typeof S!=='undefined'&&S.session)?String(S.session.session_id||''):'';
|
||
const streamId=typeof S!=='undefined'?String(S.activeStreamId||''):'';
|
||
if(!streamId) return {sessionId:sessionId,streamId:'',controller:null,promise:Promise.resolve(false)};
|
||
const controller=createAbortController();
|
||
const timer=window.setTimeout(function(){controller.abort();},1800);
|
||
const promise=(async function(){
|
||
try{
|
||
const url=new URL('api/chat/cancel?stream_id='+encodeURIComponent(streamId),document.baseURI||location.href).href;
|
||
const response=await fetch(url,{credentials:'include',signal:controller.signal});
|
||
let payload=null;
|
||
try{payload=await response.json();}catch(_){ }
|
||
const currentSession=(typeof S!=='undefined'&&S.session)?String(S.session.session_id||''):'';
|
||
if(response.ok&&payload&&payload.cancelled===false&¤tSession===sessionId&&String(S.activeStreamId||'')===streamId){
|
||
S.activeStreamId=null;
|
||
if(S.session) S.session.active_stream_id=null;
|
||
if(typeof setBusy==='function') setBusy(false); else S.busy=false;
|
||
}
|
||
return !!response.ok;
|
||
}catch(_){
|
||
return false;
|
||
}finally{
|
||
window.clearTimeout(timer);
|
||
}
|
||
})();
|
||
return {sessionId:sessionId,streamId:streamId,controller:controller,promise:promise};
|
||
}
|
||
|
||
function trimBargeLookback(monitor){
|
||
const maximum=Math.max(1,Math.round(monitor.context.sampleRate*BARGE_LOOKBACK_MS/1000));
|
||
while(monitor.lookbackFrames>maximum&&monitor.lookback.length){
|
||
const overflow=monitor.lookbackFrames-maximum;
|
||
const first=monitor.lookback[0];
|
||
if(first.length<=overflow){monitor.lookback.shift();monitor.lookbackFrames-=first.length;continue;}
|
||
monitor.lookback[0]=first.slice(overflow);
|
||
monitor.lookbackFrames-=overflow;
|
||
}
|
||
}
|
||
|
||
async function triggerBargeIn(monitor){
|
||
if(!bargeMonitor||bargeMonitor!==monitor||monitor.cancelled||!active||monitor.token!==generation) return;
|
||
bargeMonitor=null;
|
||
if(monitor.timer){window.clearInterval(monitor.timer);monitor.timer=null;}
|
||
// Keep the old capture worklet alive until the new STT worklet is attached.
|
||
// Its lookback becomes an untrimmed handoff buffer so no syllables disappear
|
||
// during AudioWorklet/session setup.
|
||
monitor.handoff=true;
|
||
const oldToken=monitor.token;
|
||
recordPendingStitch();
|
||
generation+=1;
|
||
const token=generation;
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
suppressAutoRead=true;
|
||
clearSttLanguage();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
stopPlayback();
|
||
if(typeof window.stopTTS==='function') window.stopTTS();
|
||
clearBargeCancellation();
|
||
bargeCancelPromise=cancelActiveModelTurn();
|
||
setState('listening','Listening — interrupted');
|
||
if(oldToken===token) return;
|
||
startListening(token,{
|
||
stream:monitor.stream,
|
||
context:monitor.context,
|
||
lookback:monitor.lookback,
|
||
handoffMonitor:monitor,
|
||
heardSpeech:true,
|
||
requireStreamingLookback:true,
|
||
});
|
||
}
|
||
|
||
function bargeFromLiveCapture(){
|
||
// Live-capture barge-in: the always-on VAD/STT listener detected a real
|
||
// speech onset while Hermes was mid-response. Cancel the RESPONSE side
|
||
// only — cues, playback, the model stream — and bump `generation` so every
|
||
// stale response-side continuation dies. The already-running capture is
|
||
// guarded by captureGeneration and keeps collecting the interrupting
|
||
// utterance without losing a syllable.
|
||
if(!active||(state!=='thinking'&&state!=='speaking')) return;
|
||
recordPendingStitch();
|
||
generation+=1;
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
suppressAutoRead=true;
|
||
clearSttLanguage();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
stopPlayback();
|
||
if(typeof window.stopTTS==='function') window.stopTTS();
|
||
clearBargeCancellation();
|
||
bargeCancelPromise=cancelActiveModelTurn();
|
||
setState('listening','Listening — interrupted');
|
||
}
|
||
|
||
async function startBargeMonitor(token){
|
||
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')||bargeMonitor) return;
|
||
const monitor={token:token,cancelled:false,handoff:false,aecUsable:true,timer:null,stream:null,context:null,source:null,node:null,silentGain:null,lookback:[],lookbackFrames:0};
|
||
bargeMonitor=monitor;
|
||
try{
|
||
monitor.stream=await acquireMicrophone();
|
||
if(!active||token!==generation||bargeMonitor!==monitor){disposeBargeResources(monitor,false);return;}
|
||
monitor.aecUsable=captureAecIsUsable(monitor.stream);
|
||
const Context=window.AudioContext||window.webkitAudioContext;
|
||
try{monitor.context=new Context({latencyHint:'interactive'});}catch(_){monitor.context=new Context();}
|
||
const analyser=monitor.context.createAnalyser();
|
||
analyser.fftSize=1024;
|
||
monitor.source=monitor.context.createMediaStreamSource(monitor.stream);
|
||
monitor.source.connect(analyser);
|
||
if(monitor.context.audioWorklet&&window.AudioWorkletNode){
|
||
await monitor.context.audioWorklet.addModule(WORKLET_URL);
|
||
if(bargeMonitor!==monitor||monitor.cancelled){disposeBargeResources(monitor,false);return;}
|
||
monitor.node=new AudioWorkletNode(monitor.context,'atlas-pcm-capture');
|
||
monitor.silentGain=monitor.context.createGain();
|
||
monitor.silentGain.gain.value=0;
|
||
monitor.node.port.onmessage=function(event){
|
||
if(monitor.cancelled||(!monitor.handoff&&bargeMonitor!==monitor)||!event.data||event.data.type!=='pcm') return;
|
||
const samples=new Float32Array(event.data.samples);
|
||
monitor.lookback.push(samples);
|
||
monitor.lookbackFrames+=samples.length;
|
||
if(!monitor.handoff) trimBargeLookback(monitor);
|
||
};
|
||
monitor.source.connect(monitor.node);
|
||
monitor.node.connect(monitor.silentGain);
|
||
monitor.silentGain.connect(monitor.context.destination);
|
||
}
|
||
await monitor.context.resume();
|
||
const samples=new Uint8Array(analyser.fftSize);
|
||
let noiseFloor=0.008;
|
||
let voiceFrames=0;
|
||
let ducked=false;
|
||
let speechArmAt=Date.now()+100;
|
||
let playbackWasActive=false;
|
||
monitor.timer=window.setInterval(function(){
|
||
if(!active||token!==generation||bargeMonitor!==monitor||(state!=='thinking'&&state!=='speaking')){stopBargeMonitor();return;}
|
||
analyser.getByteTimeDomainData(samples);
|
||
let energy=0;
|
||
for(let index=0;index<samples.length;index+=1){const value=(samples[index]-128)/128;energy+=value*value;}
|
||
const rms=Math.sqrt(energy/samples.length);
|
||
const now=Date.now();
|
||
const playbackActive=playbackAudible();
|
||
if(playbackActive&&!playbackWasActive){
|
||
// AEC needs a brief convergence window when local speech starts. PCM
|
||
// lookback preserves a real interruption spoken during this guard.
|
||
// A brief cached thinking cue uses the shorter guard so an "um" can
|
||
// still interrupt it; answer playback gets the conservative window.
|
||
speechArmAt=now+(state==='thinking'?150:450);
|
||
voiceFrames=0;
|
||
if(ducked){ducked=false;setPlaybackDucked(false);}
|
||
}else if(!playbackActive&&playbackWasActive){
|
||
speechArmAt=now+100;
|
||
}
|
||
playbackWasActive=playbackActive;
|
||
const threshold=Math.max(0.05,(noiseFloor*3)+0.008);
|
||
// Never treat speaker leakage as an interruption when the browser says
|
||
// the requested AEC was not applied. Quiet Thinking remains fully
|
||
// interruptible, and unknown/unsupported settings preserve normal use.
|
||
const automaticBargeAllowed=!playbackActive||monitor.aecUsable;
|
||
const voiceNow=automaticBargeAllowed&&now>=speechArmAt&&rms>threshold;
|
||
if(!voiceNow) noiseFloor=(noiseFloor*0.975)+(Math.min(rms,threshold)*0.025);
|
||
voiceFrames=voiceNow?Math.min(voiceFrames+1,BARGE_TRIGGER_FRAMES):Math.max(voiceFrames-1,0);
|
||
if(voiceFrames>=BARGE_DUCK_FRAMES&&!ducked){ducked=true;setPlaybackDucked(true);}
|
||
if(!voiceFrames&&ducked){ducked=false;setPlaybackDucked(false);}
|
||
if(voiceFrames>=BARGE_TRIGGER_FRAMES) triggerBargeIn(monitor);
|
||
},50);
|
||
}catch(_){
|
||
if(bargeMonitor===monitor) bargeMonitor=null;
|
||
disposeBargeResources(monitor,false);
|
||
// Barge-in is an optional full-duplex enhancement. The normal finalized
|
||
// microphone turn remains available if the browser rejects concurrent
|
||
// capture or AudioWorklet initialization.
|
||
}
|
||
}
|
||
|
||
async function transcribeStreamingOrFallback(blob,token,session,allowContainerFallback,turnId){
|
||
if(!active||token!==generation) return;
|
||
setState('transcribing');
|
||
let pcmFallback=null;
|
||
if(session){
|
||
try{
|
||
const result=await session.commit();
|
||
if(!String(result.transcript||'').trim()) throw new Error('Streaming transcription returned no final text');
|
||
if(active&&token===generation){
|
||
if(streamingStt===session) streamingStt=null;
|
||
sendTranscript(result.transcript,token,result.language,turnId);
|
||
return;
|
||
}
|
||
}catch(_){
|
||
// The finalized browser container below is the quality-preserving path
|
||
// whenever rolling PCM transport or speculative inference fails.
|
||
if(typeof session.takeFallbackBlob==='function') pcmFallback=session.takeFallbackBlob();
|
||
}
|
||
if(streamingStt===session) streamingStt=null;
|
||
}
|
||
if(pcmFallback){
|
||
transcribe(pcmFallback,token,turnId);
|
||
return;
|
||
}
|
||
if(allowContainerFallback===false){
|
||
toast('I missed the start of that interruption. Please repeat it.');
|
||
restartSoon(token,250);
|
||
return;
|
||
}
|
||
transcribe(blob,token,turnId);
|
||
}
|
||
|
||
async function startListening(token,reusedCapture){
|
||
if(!active||token!==generation) return;
|
||
stopCapture();
|
||
// A continuous-capture restart (reusedCapture.preserveDisplay) happens the
|
||
// instant an utterance finalizes, while the display legitimately shows
|
||
// Hermes's own activity (transcribing/thinking/speaking). Capture is
|
||
// therefore tracked by captureActive, never by the visible state, and the
|
||
// response-side teardown below is skipped so that turn is not disturbed.
|
||
const preserveDisplay=!!(reusedCapture&&reusedCapture.preserveDisplay);
|
||
if(!preserveDisplay){
|
||
stopPlayback();
|
||
cancelSpeechTurn();
|
||
clearSttLanguage();
|
||
}
|
||
captureTurnId=(voiceTabNonce?voiceTabNonce+'-':'')+String(token)+'-'+String(++turnSequence);
|
||
// Capture turns carry their own epoch: a response-side barge bumps
|
||
// `generation` but never this counter, so capture survives it seamlessly.
|
||
const captureToken=++captureGeneration;
|
||
captureActive=true;
|
||
if(!preserveDisplay) setState('listening');
|
||
try{
|
||
// A barge-monitor handoff supplies its own stream/context; otherwise the
|
||
// retained session microphone is reused. Only a fresh hands-free session
|
||
// actually asks for the microphone again, keeping acquireMicrophone()
|
||
// the single getUserMedia call site.
|
||
if(reusedCapture&&reusedCapture.stream&&stream&&reusedCapture.stream!==stream){
|
||
stream.getTracks().forEach(function(track){track.stop();});
|
||
stream=null;
|
||
}
|
||
if(reusedCapture&&reusedCapture.context&&audioContext&&reusedCapture.context!==audioContext){
|
||
try{audioContext.close();}catch(_){ }
|
||
audioContext=null;
|
||
}
|
||
const capture=(reusedCapture&&reusedCapture.stream)||stream||await acquireMicrophone();
|
||
if(!active||captureToken!==captureGeneration){
|
||
if(reusedCapture&&reusedCapture.handoffMonitor) disposeBargeResources(reusedCapture.handoffMonitor,false);
|
||
else if(capture!==stream) capture.getTracks().forEach(function(track){track.stop();});
|
||
return;
|
||
}
|
||
stream=capture;
|
||
applyConversationMute();
|
||
const captureAec=captureAecIsUsable(stream);
|
||
const Context=window.AudioContext||window.webkitAudioContext;
|
||
if(reusedCapture&&reusedCapture.context){
|
||
audioContext=reusedCapture.context;
|
||
}else if(!audioContext){
|
||
try{
|
||
// Let the browser's native resampler produce Whisper's 16 kHz input.
|
||
audioContext=new Context({sampleRate:16000,latencyHint:'interactive'});
|
||
}catch(_){
|
||
audioContext=new Context();
|
||
}
|
||
}
|
||
const analyser=audioContext.createAnalyser();
|
||
analyser.fftSize=2048;
|
||
const highpass=audioContext.createBiquadFilter();
|
||
highpass.type='highpass';
|
||
highpass.frequency.value=140;
|
||
highpass.Q.value=0.7;
|
||
const mediaSource=audioContext.createMediaStreamSource(stream);
|
||
mediaSource.connect(highpass);
|
||
highpass.connect(analyser);
|
||
captureGraph={mediaSource:mediaSource,highpass:highpass,analyser:analyser,silentGain:null};
|
||
streamingStt=createStreamingSttSession(captureTurnId,audioContext);
|
||
// Whisper receives the browser's full-band processed microphone signal;
|
||
// the 140 Hz high-pass remains a VAD-only aid so low voices are not
|
||
// needlessly altered before final recognition.
|
||
if(streamingStt) await installCaptureWorklet(audioContext,mediaSource,streamingStt);
|
||
if(streamingStt&&reusedCapture&&Array.isArray(reusedCapture.lookback)){
|
||
reusedCapture.lookback.forEach(function(samples){streamingStt.push(samples);});
|
||
}
|
||
if(reusedCapture&&reusedCapture.handoffMonitor){
|
||
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
||
reusedCapture.handoffMonitor=null;
|
||
}
|
||
if(reusedCapture&&reusedCapture.requireStreamingLookback&&!streamingStt){
|
||
toast('Streaming transcription is unavailable. Please repeat your interruption.');
|
||
stopCapture();
|
||
restartSoon(token,250);
|
||
return;
|
||
}
|
||
const samples=new Uint8Array(analyser.fftSize);
|
||
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm'];
|
||
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
||
const chunks=[];
|
||
let heardSpeech=!!(reusedCapture&&reusedCapture.heardSpeech);
|
||
let voiceFrames=heardSpeech?3:0;
|
||
// Voiced milliseconds collected this utterance. A barge handoff already
|
||
// carries confirmed speech, so it starts past the young-utterance hold.
|
||
let speechMs=heardSpeech?VAD_COMMITTED_SPEECH_MS:0;
|
||
let noiseFloor=0.008;
|
||
let lastSpeech=Date.now();
|
||
let speculative=false;
|
||
let ducked=false;
|
||
let bargeArmAt=0;
|
||
let playbackWasLive=false;
|
||
const started=Date.now();
|
||
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
||
let recordedMime=recorder.mimeType||mime||'';
|
||
recorder.ondataavailable=function(event){
|
||
if(!event.data||!event.data.size) return;
|
||
if(event.data.type) recordedMime=event.data.type;
|
||
chunks.push(event.data);
|
||
};
|
||
recorder.onstop=async function(){
|
||
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
||
const finalCaptureNode=captureNode;
|
||
if(finalCaptureNode&&typeof finalCaptureNode._atlasFlush==='function'){
|
||
await Promise.race([
|
||
finalCaptureNode._atlasFlush(),
|
||
new Promise(function(resolve){window.setTimeout(resolve,100);}),
|
||
]);
|
||
}
|
||
// A stale onstop (teardown by stopCapture/releaseMicrophone, or a
|
||
// newer capture turn already running) must not touch the globals: by
|
||
// now they can belong to the NEXT turn, and its session was already
|
||
// cancelled by stopCapture. Real recorders fire onstop asynchronously,
|
||
// so this guard has to come before anything else is read.
|
||
if(!active||captureToken!==captureGeneration) return;
|
||
// Continuous capture: the microphone stream and AudioContext stay hot.
|
||
// Detach only this utterance's session, then re-enter capture below so
|
||
// speech during transcribing/thinking becomes the next utterance.
|
||
const utteranceTurnId=captureTurnId;
|
||
const session=streamingStt;
|
||
streamingStt=null;
|
||
recorder=null;
|
||
if(!heardSpeech||!chunks.length){
|
||
if(session) session.cancel();
|
||
// Nothing worth transcribing: recycle the recorder on the hot mic.
|
||
startListening(generation,{preserveDisplay:true});
|
||
return;
|
||
}
|
||
// The utterance is dispatched under the CURRENT response epoch: when a
|
||
// live-capture barge just cancelled a model turn, this transcript is
|
||
// the interruption that replaces it.
|
||
transcribeStreamingOrFallback(
|
||
new Blob(chunks,{type:recordedMime||'audio/webm'}),
|
||
generation,
|
||
session,
|
||
!(reusedCapture&&reusedCapture.requireStreamingLookback),
|
||
utteranceTurnId
|
||
);
|
||
// Re-enter capture immediately — never wait for transcription or the
|
||
// response. The display may show transcribing/thinking while the next
|
||
// capture turn is already live underneath.
|
||
startListening(generation,{preserveDisplay:true});
|
||
};
|
||
// Ask the browser for one finalized container at stop. Android Chromium
|
||
// can emit timeslice fragments without a reusable EBML initialization
|
||
// header; concatenating those fragments made otherwise valid recordings
|
||
// intermittently unreadable by ffmpeg. A bounded 90-second Opus capture
|
||
// is small enough to retain as one browser-owned recording.
|
||
recorder.start();
|
||
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1100',10)||1100);
|
||
const speculateMs=Math.min(silenceMs-250,Math.max(450,Math.round(silenceMs*0.55)));
|
||
vadTimer=window.setInterval(function(){
|
||
if(!active||captureToken!==captureGeneration||!recorder||recorder.state==='inactive') return;
|
||
analyser.getByteTimeDomainData(samples);
|
||
let energy=0;
|
||
for(let index=0;index<samples.length;index+=1){
|
||
const normalized=(samples[index]-128)/128;
|
||
energy+=normalized*normalized;
|
||
}
|
||
const rms=Math.sqrt(energy/samples.length);
|
||
updateInputLevel(rms);
|
||
const now=Date.now();
|
||
// Continuous capture stays live while Hermes thinks and speaks, so the
|
||
// same VAD that endpoints utterances is also the barge-in detector.
|
||
// While our own audio plays the microphone hears the speakers: without
|
||
// AEC nothing counts as speech at all, and with AEC a post-playback
|
||
// arming delay plus the sustained-frame requirement below mirror the
|
||
// conservative energy monitor this path supersedes.
|
||
const playbackLive=playbackAudible();
|
||
if(playbackLive&&!playbackWasLive){
|
||
bargeArmAt=now+(state==='thinking'?150:450);
|
||
}else if(!playbackLive&&playbackWasLive){
|
||
bargeArmAt=now+100;
|
||
}
|
||
playbackWasLive=playbackLive;
|
||
setConversationPlaying(playbackLive);
|
||
const speechThreshold=Math.max(0.04,noiseFloor*2.4+0.006);
|
||
const voiceNow=rms>speechThreshold&&(!playbackLive||(captureAec&&now>=bargeArmAt));
|
||
// Adapt the floor only toward energies at or below the current
|
||
// threshold (like the fallback energy monitor): echo residue during
|
||
// continuous capture must never ratchet the threshold above real
|
||
// speech and deafen onset detection for the next utterance.
|
||
if(!heardSpeech&&!voiceNow){noiseFloor=(noiseFloor*0.94)+(Math.min(rms,speechThreshold)*0.06);}
|
||
voiceFrames=voiceNow?Math.min(voiceFrames+1,5):Math.max(voiceFrames-1,0);
|
||
if(voiceNow) speechMs+=100;
|
||
if(playbackLive){
|
||
if(voiceFrames>=BARGE_DUCK_FRAMES&&!ducked){ducked=true;setPlaybackDucked(true);}
|
||
if(!voiceFrames&&ducked){ducked=false;setPlaybackDucked(false);}
|
||
}else if(ducked){
|
||
ducked=false;
|
||
setPlaybackDucked(false);
|
||
}
|
||
// Speech onset. While playback runs an echo-safe barge needs the same
|
||
// sustained evidence (BARGE_TRIGGER_FRAMES) as the old energy monitor.
|
||
const onsetFrames=playbackLive?BARGE_TRIGGER_FRAMES:3;
|
||
if(!heardSpeech&&voiceFrames>=onsetFrames){
|
||
heardSpeech=true;
|
||
lastSpeech=now;
|
||
// The user started a new utterance while Hermes was mid-response:
|
||
// cancel the response side only; this capture keeps running.
|
||
if(state==='thinking'||state==='speaking') bargeFromLiveCapture();
|
||
}else if(heardSpeech&&voiceNow){
|
||
const speechGapMs=now-lastSpeech;
|
||
lastSpeech=now;
|
||
// Resume after our own speculate AND after any gap long enough for
|
||
// the server's independent 650ms end-of-speech detector: either
|
||
// party may have frozen a partial-utterance snapshot by now.
|
||
if((speculative||speechGapMs>=SERVER_EOS_SILENCE_MS)&&streamingStt){streamingStt.resume();speculative=false;}
|
||
}
|
||
if(heardSpeech&&!voiceNow&&!speculative&&speechMs>=SPECULATE_MIN_SPEECH_MS&&(now-lastSpeech)>=speculateMs&&streamingStt){
|
||
streamingStt.speculate();
|
||
speculative=true;
|
||
}
|
||
// A young utterance holds its endpoint longer: pausing to think right
|
||
// after the first word must not send a one-word fragment. When the
|
||
// streaming partial already reads as a plausibly complete utterance
|
||
// (>=3 words, or terminal punctuation), endpoint at the base window
|
||
// instead so short commands are not delayed by the clipping guard —
|
||
// the long hold remains only for 1-2 word partials.
|
||
const partialText=streamingStt&&streamingStt.latestPartial?streamingStt.latestPartial():'';
|
||
const partialWords=partialText?partialText.split(/\s+/).filter(Boolean).length:0;
|
||
const partialComplete=partialWords>=3||(partialWords>0&&/[.!?…]["')\]}]*$/.test(partialText));
|
||
const endpointSilenceMs=(speechMs<VAD_COMMITTED_SPEECH_MS&&!partialComplete)?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs;
|
||
const finished=heardSpeech&&(now-lastSpeech)>=endpointSilenceMs;
|
||
const timedOut=now-started>=90000;
|
||
const idle=(!heardSpeech)&&(now-started)>=20000;
|
||
if(finished||timedOut||idle){
|
||
if(vadTimer){clearInterval(vadTimer);vadTimer=null;}
|
||
try{recorder.stop();}catch(_){ }
|
||
}
|
||
},100);
|
||
}catch(error){
|
||
if(reusedCapture&&reusedCapture.handoffMonitor){
|
||
disposeBargeResources(reusedCapture.handoffMonitor,true);
|
||
reusedCapture.handoffMonitor=null;
|
||
}
|
||
if(!active||captureToken!==captureGeneration) return;
|
||
// This capture turn is dead. A fatal microphone error ends the session
|
||
// and releases the retained microphone via showUnavailable().
|
||
captureActive=false;
|
||
const message=errorMessage(error,'Microphone permission is required');
|
||
showUnavailable(message);
|
||
toast(message);
|
||
}
|
||
}
|
||
|
||
function stripHttpUrlsForSpeech(text){
|
||
return String(text||'').replace(/(^|\s+[([{]|\s+|[([{"'])https?:\/\/[^\s<>"']+/gi,function(match,prefix){
|
||
let address=match.slice(prefix.length);
|
||
let suffix='';
|
||
while(address){
|
||
const last=address.slice(-1);
|
||
let trailing=/[.,!?;:…，。！？；：]/.test(last);
|
||
if(last===')') trailing=(address.match(/\)/g)||[]).length>(address.match(/\(/g)||[]).length;
|
||
if(last===']') trailing=(address.match(/\]/g)||[]).length>(address.match(/\[/g)||[]).length;
|
||
if(last==='}') trailing=(address.match(/\}/g)||[]).length>(address.match(/\{/g)||[]).length;
|
||
if(!trailing) break;
|
||
suffix=last+suffix;
|
||
address=address.slice(0,-1);
|
||
}
|
||
const pairs={'(':')','[':']','{':'}'};
|
||
const opening=prefix.slice(-1);
|
||
if(pairs[opening]&&suffix.startsWith(pairs[opening])){
|
||
prefix=prefix.slice(0,-1);
|
||
suffix=suffix.slice(1);
|
||
}
|
||
if(/^\s+$/.test(prefix)&&/^[.,!?;:…，。！？；：]/.test(suffix)) prefix='';
|
||
return prefix+suffix;
|
||
});
|
||
}
|
||
|
||
window._atlasStripHttpUrlsForSpeech=stripHttpUrlsForSpeech;
|
||
|
||
function cleanForSpeech(text){
|
||
const cleaned=typeof window._stripForTTS==='function'?window._stripForTTS(text):String(text||'').replace(/```[\s\S]*?```/g,' code block ');
|
||
return stripHttpUrlsForSpeech(cleaned).replace(/\s+/g,' ').trim();
|
||
}
|
||
|
||
function sentenceEnd(text){
|
||
const source=String(text||'');
|
||
const boundary=/[.!?…]+["')\]}]*(?:\s|$)/g;
|
||
const abbreviation=/(?:\b(?:mr|mrs|ms|dr|prof|sr|jr|st|vs|etc|e\.g|i\.e)|\b[A-Z])\.$/i;
|
||
let match;
|
||
while((match=boundary.exec(source))){
|
||
const end=match.index+match[0].trimEnd().length;
|
||
const prefix=source.slice(0,end).replace(/["')\]}]+$/,'');
|
||
if(prefix.endsWith('.')&&abbreviation.test(prefix)) continue;
|
||
return end;
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
function preferredCut(text,min,target,max){
|
||
const bounded=text.slice(0,max);
|
||
const candidates=[];
|
||
const punctuation=/[,;:—–.!?…]+["')\]}]*(?:\s|$)/g;
|
||
let match;
|
||
while((match=punctuation.exec(bounded))){
|
||
const end=match.index+match[0].trimEnd().length;
|
||
if(end>=min) candidates.push({end:end,weight:Math.abs(end-target)});
|
||
}
|
||
if(candidates.length){
|
||
candidates.sort(function(left,right){return left.weight-right.weight;});
|
||
return candidates[0].end;
|
||
}
|
||
const spaces=[];
|
||
const whitespace=/\s+/g;
|
||
while((match=whitespace.exec(bounded))){if(match.index>=min) spaces.push(match.index);}
|
||
if(spaces.length){
|
||
spaces.sort(function(left,right){return Math.abs(left-target)-Math.abs(right-target);});
|
||
return spaces[0];
|
||
}
|
||
return Math.min(max,text.length);
|
||
}
|
||
|
||
function adaptiveChunks(text,final,firstChunk){
|
||
const source=String(text||'').trim();
|
||
if(!source) return {chunks:[],consumed:0};
|
||
const chunks=[];
|
||
let offset=0;
|
||
if(firstChunk!==false){
|
||
const firstSentence=sentenceEnd(source);
|
||
if(firstSentence<0&&!final) return {chunks:[],consumed:0};
|
||
const firstAvailable=firstSentence>=0?firstSentence:source.length;
|
||
const firstSlice=source.slice(0,firstAvailable);
|
||
const firstCut=firstSlice.length<=60?firstSlice.length:preferredCut(firstSlice,40,52,60);
|
||
chunks.push(source.slice(0,firstCut).trim());
|
||
offset=firstCut;
|
||
}
|
||
while(offset<source.length){
|
||
const remaining=source.slice(offset).trimStart();
|
||
offset=source.length-remaining.length;
|
||
if(!remaining) break;
|
||
if(remaining.length<100){
|
||
const complete=sentenceEnd(remaining);
|
||
if(!final&&complete<0) break;
|
||
const end=final?remaining.length:complete;
|
||
chunks.push(remaining.slice(0,end).trim());
|
||
offset+=end;
|
||
continue;
|
||
}
|
||
const cut=preferredCut(remaining,100,120,140);
|
||
if(!final&&cut===remaining.length&&remaining.length<140) break;
|
||
chunks.push(remaining.slice(0,cut).trim());
|
||
offset+=cut;
|
||
}
|
||
return {chunks:chunks.filter(Boolean),consumed:offset};
|
||
}
|
||
|
||
// Export the deterministic splitter as a narrow diagnostic/test seam.
|
||
window._atlasAdaptiveChunks=function(text,final){return adaptiveChunks(text,final!==false,true).chunks;};
|
||
|
||
function playBlob(blob,token){
|
||
return new Promise(function(resolve,reject){
|
||
if(!active||token!==generation){resolve();return;}
|
||
const session=playbackSession;
|
||
const url=URL.createObjectURL(blob);
|
||
const audio=new Audio(url);
|
||
currentAudio=audio;
|
||
let settled=false;
|
||
function cleanup(callback,value){
|
||
if(settled) return;
|
||
settled=true;
|
||
if(currentAudio===audio) currentAudio=null;
|
||
if(session&&session.blobWake===cancel) session.blobWake=null;
|
||
indicator.classList.remove('is-playing');
|
||
URL.revokeObjectURL(url);
|
||
callback(value);
|
||
}
|
||
function cancel(){cleanup(resolve);}
|
||
if(session) session.blobWake=cancel;
|
||
audio.onended=function(){cleanup(resolve);};
|
||
audio.onerror=function(){cleanup(reject,new Error('Local speech playback failed'));};
|
||
audio.play().then(function(){
|
||
if(active&&token===generation&¤tAudio===audio) indicator.classList.add('is-playing');
|
||
}).catch(function(error){cleanup(reject,error);});
|
||
});
|
||
}
|
||
|
||
function ttsSpeed(){
|
||
// Hands-free speech-rate preference. Piper accepts 0.5-2.0 (length_scale
|
||
// 1/speed, no pitch shift); hands-free defaults to a slightly brisk 1.15,
|
||
// seeded once at initialize() and user-tunable via localStorage.
|
||
const stored=parseFloat(localStorage.getItem('hermes-voice-tts-speed')||'');
|
||
if(!Number.isFinite(stored)) return TTS_SPEED_DEFAULT;
|
||
return Math.min(TTS_SPEED_MAX,Math.max(TTS_SPEED_MIN,stored));
|
||
}
|
||
|
||
function ttsRequest(chunk,language,turnId){
|
||
const request={text:chunk,engine:'atlas',turn_id:turnId,speed:ttsSpeed()};
|
||
if(language) request.language=language;
|
||
return request;
|
||
}
|
||
|
||
async function fetchSpeech(chunk,language,turnId,token){
|
||
// `language` is only ever the private STT result for this turn. When it is
|
||
// absent the field is omitted entirely and the server picks English.
|
||
if(!active||token!==generation) throw cancelledError('Speech turn cancelled');
|
||
const session=playbackSession;
|
||
if(!session||session.cancelled||session.turnId!==turnId) throw cancelledError('Speech turn cancelled');
|
||
const controller=createAbortController();
|
||
session.controllers.add(controller);
|
||
try{
|
||
const response=await fetch('/api/tts',{
|
||
method:'POST',
|
||
headers:{'Content-Type':'application/json'},
|
||
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
||
signal:controller.signal,
|
||
});
|
||
if(!response.ok){
|
||
const payload=await response.json().catch(function(){return {};});
|
||
throw new Error(payload.error||('Local speech request failed: '+response.status));
|
||
}
|
||
const blob=await response.blob();
|
||
if(!active||token!==generation||session.cancelled||playbackSession!==session) throw cancelledError('Speech turn cancelled');
|
||
return blob;
|
||
}finally{
|
||
session.controllers.delete(controller);
|
||
}
|
||
}
|
||
|
||
function cuePoolOffset(turnId,length){
|
||
let hash=0;
|
||
const source=String(turnId||'');
|
||
for(let index=0;index<source.length;index+=1) hash=((hash*31)+source.charCodeAt(index))>>>0;
|
||
return length?hash%length:0;
|
||
}
|
||
|
||
function thinkingCueStillOwned(cue){
|
||
return !!(cue&&!cue.cancelled&&thinkingCue===cue&&active&&cue.token===generation&&state==='thinking');
|
||
}
|
||
|
||
function scheduleNextThinkingCue(cue,delay){
|
||
if(!thinkingCueStillOwned(cue)||cue.issued>=cue.pool.length) return;
|
||
cue.timer=window.setTimeout(function(){
|
||
cue.timer=null;
|
||
issueThinkingCue(cue);
|
||
},delay);
|
||
}
|
||
|
||
async function issueThinkingCue(cue){
|
||
if(!thinkingCueStillOwned(cue)||cue.issued>=cue.pool.length) return;
|
||
const entry=cue.pool[(cue.offset+cue.issued)%cue.pool.length];
|
||
const cueNumber=cue.issued+1;
|
||
cue.issued+=1;
|
||
const controller=createAbortController();
|
||
cue.controller=controller;
|
||
try{
|
||
const request=ttsRequest(entry.text,cue.language,cue.turnId+':thinking-cue:'+cueNumber);
|
||
request.cue_id=entry.id;
|
||
const response=await fetch(TTS_STREAM_URL,{
|
||
method:'POST',
|
||
headers:{'Content-Type':'application/json','Accept':'audio/pcm'},
|
||
body:JSON.stringify(request),
|
||
signal:controller.signal,
|
||
});
|
||
if(!response.ok||!response.body) throw new Error('Cached thinking cue unavailable');
|
||
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||'22050',10);
|
||
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000) throw new Error('Cached thinking cue metadata invalid');
|
||
if(!thinkingCueStillOwned(cue)) return;
|
||
const Context=window.AudioContext||window.webkitAudioContext;
|
||
try{cue.context=new Context({sampleRate:sampleRate,latencyHint:'interactive'});}catch(_){cue.context=new Context();}
|
||
await cue.context.audioWorklet.addModule(WORKLET_URL);
|
||
if(!thinkingCueStillOwned(cue)) return;
|
||
cue.node=new AudioWorkletNode(cue.context,'atlas-pcm-playback');
|
||
cue.gain=cue.context.createGain();
|
||
cue.node.connect(cue.gain);
|
||
cue.gain.connect(cue.context.destination);
|
||
const drained=new Promise(function(resolve){
|
||
cue.audioWake=resolve;
|
||
cue.node.port.onmessage=function(event){
|
||
if((event.data||{}).type==='drained'){
|
||
const wake=cue.audioWake;
|
||
cue.audioWake=null;
|
||
if(wake) wake();
|
||
}
|
||
};
|
||
});
|
||
await cue.context.resume();
|
||
indicator.classList.add('is-playing');
|
||
const reader=response.body.getReader();
|
||
let carry=null;
|
||
while(thinkingCueStillOwned(cue)){
|
||
const result=await reader.read();
|
||
if(result.done) break;
|
||
let bytes=result.value;
|
||
if(carry!==null){const joined=new Uint8Array(bytes.length+1);joined[0]=carry;joined.set(bytes,1);bytes=joined;carry=null;}
|
||
if(bytes.length%2){carry=bytes[bytes.length-1];bytes=bytes.slice(0,-1);}
|
||
if(!bytes.length) continue;
|
||
const copy=bytes.buffer.slice(bytes.byteOffset,bytes.byteOffset+bytes.byteLength);
|
||
cue.node.port.postMessage({type:'push',samples:copy,sampleRate:sampleRate},[copy]);
|
||
}
|
||
if(thinkingCueStillOwned(cue)){
|
||
cue.node.port.postMessage({type:'end'});
|
||
await drained;
|
||
}else{
|
||
try{await reader.cancel();}catch(_){ }
|
||
}
|
||
cue.controller=null;
|
||
if(cue.node){try{cue.node.disconnect();}catch(_){ }cue.node=null;}
|
||
if(cue.context){try{cue.context.close();}catch(_){ }cue.context=null;}
|
||
cue.gain=null;
|
||
indicator.classList.remove('is-playing');
|
||
if(thinkingCueStillOwned(cue)) scheduleNextThinkingCue(cue,THINKING_CUE_INTERVAL_MS);
|
||
}catch(error){
|
||
cue.controller=null;
|
||
if(cue.cancelled||(error&&error.name==='AbortError')) return;
|
||
// Cues are optional. A playback or synthesis failure must not disturb the
|
||
// answer, retry noisily, or disable the primary hands-free conversation.
|
||
if(thinkingCue===cue) cancelThinkingCues();
|
||
}
|
||
}
|
||
|
||
function scheduleThinkingCues(token,language,turnId){
|
||
cancelThinkingCues();
|
||
if(!active||token!==generation||state!=='thinking') return;
|
||
// Missing/unknown is the same fail-safe route as answer TTS: English/Amy.
|
||
// Known STT languages remain bound exactly to their turn voice.
|
||
const localized=normalizeSttLanguage(language)||'en';
|
||
const pool=THINKING_CUE_POOLS[localized];
|
||
const cue={
|
||
token:token,
|
||
turnId:String(turnId||token),
|
||
language:localized,
|
||
pool:pool,
|
||
offset:cuePoolOffset(turnId,pool.length),
|
||
issued:0,
|
||
timer:null,
|
||
controller:null,
|
||
audioWake:null,
|
||
context:null,
|
||
node:null,
|
||
gain:null,
|
||
cancelled:false,
|
||
};
|
||
thinkingCue=cue;
|
||
scheduleNextThinkingCue(cue,THINKING_CUE_FIRST_MS);
|
||
}
|
||
|
||
async function prepareSpeech(chunk,language,turnId,token){
|
||
if(streamingCapability.tts&&window.ReadableStream&&window.AudioWorkletNode){
|
||
const controller=createAbortController();
|
||
if(playbackSession) playbackSession.controllers.add(controller);
|
||
try{
|
||
const response=await fetch(TTS_STREAM_URL,{
|
||
method:'POST',
|
||
headers:{'Content-Type':'application/json','Accept':'audio/pcm'},
|
||
body:JSON.stringify(ttsRequest(chunk,language,turnId)),
|
||
signal:controller.signal,
|
||
});
|
||
if(!response.ok||!response.body) throw new Error('Streaming speech unavailable');
|
||
const sampleRate=parseInt(response.headers.get('X-Audio-Sample-Rate')||String(streamingCapability.tts.sample_rate||22050),10);
|
||
const channels=parseInt(response.headers.get('X-Audio-Channels')||'1',10);
|
||
if(!Number.isFinite(sampleRate)||sampleRate<8000||sampleRate>96000||channels!==1) throw new Error('Unsupported streaming speech format');
|
||
return {kind:'pcm',response:response,sampleRate:sampleRate,controller:controller,token:token,chunk:chunk,language:language,turnId:turnId,started:false};
|
||
}catch(error){
|
||
if(playbackSession) playbackSession.controllers.delete(controller);
|
||
if(error&&error.name==='AbortError') throw error;
|
||
// Disable only this optional transport for the browser session. The
|
||
// complete WAV endpoint preserves voice quality and availability.
|
||
streamingCapability.tts=null;
|
||
}
|
||
}
|
||
return {kind:'blob',chunk:chunk,blob:await fetchSpeech(chunk,language,turnId,token)};
|
||
}
|
||
|
||
async function ensurePcmPlayback(asset,token){
|
||
if(!active||token!==generation) return null;
|
||
const session=playbackSession;
|
||
if(!session||session.cancelled) return null;
|
||
if(session.node){
|
||
if(session.sampleRate!==asset.sampleRate) throw new Error('Streaming speech sample rate changed mid-turn');
|
||
return session;
|
||
}
|
||
const Context=window.AudioContext||window.webkitAudioContext;
|
||
let context;
|
||
try{
|
||
// Keep Piper at its native rate and let the browser/audio device own any
|
||
// final hardware conversion; the worklet interpolator is a fallback.
|
||
context=new Context({sampleRate:asset.sampleRate,latencyHint:'interactive'});
|
||
}catch(_){
|
||
context=new Context();
|
||
}
|
||
await context.audioWorklet.addModule(WORKLET_URL);
|
||
if(!active||token!==generation){context.close();return;}
|
||
const node=new AudioWorkletNode(context,'atlas-pcm-playback');
|
||
if(!session||session.cancelled){context.close();return;}
|
||
const gain=context.createGain();
|
||
session.context=context;
|
||
session.node=node;
|
||
session.gain=gain;
|
||
session.sampleRate=asset.sampleRate;
|
||
session.bufferedFrames=0;
|
||
session.playbackEnded=false;
|
||
node.connect(gain);
|
||
gain.connect(context.destination);
|
||
await context.resume();
|
||
session.drained=new Promise(function(resolve,reject){
|
||
session.drainWake=resolve;
|
||
session.drainReject=reject;
|
||
});
|
||
indicator.classList.add('is-playing');
|
||
node.port.onmessage=function(event){
|
||
const data=event.data||{};
|
||
if(data.type==='buffer'){
|
||
session.bufferedFrames=data.frames||0;
|
||
if(session.bufferedFrames<session.sampleRate&&session.lowWaterWake){
|
||
const wake=session.lowWaterWake;
|
||
session.lowWaterWake=null;
|
||
wake();
|
||
}
|
||
}else if(data.type==='drained'){
|
||
session.playbackEnded=true;
|
||
const wake=session.drainWake;
|
||
session.drainWake=null;
|
||
if(wake) wake();
|
||
}else if(data.type==='error'){
|
||
const reject=session.drainReject;
|
||
session.drainReject=null;
|
||
if(reject) reject(new Error(data.error||'Streaming speech playback failed'));
|
||
}
|
||
};
|
||
return session;
|
||
}
|
||
|
||
async function playPcm(asset,token){
|
||
const session=await ensurePcmPlayback(asset,token);
|
||
if(!session) return;
|
||
const reader=asset.response.body.getReader();
|
||
let carry=null;
|
||
try{
|
||
while(active&&token===generation&&!session.cancelled){
|
||
if(session.bufferedFrames>asset.sampleRate*2){
|
||
await new Promise(function(resolve){
|
||
let completed=false;
|
||
const wake=function(){if(completed)return;completed=true;session.lowWaterWake=null;resolve();};
|
||
session.lowWaterWake=wake;
|
||
window.setTimeout(wake,3000);
|
||
});
|
||
}
|
||
const result=await reader.read();
|
||
if(result.done) break;
|
||
let bytes=result.value;
|
||
if(carry!==null){
|
||
const joined=new Uint8Array(bytes.length+1);
|
||
joined[0]=carry;joined.set(bytes,1);bytes=joined;carry=null;
|
||
}
|
||
if(bytes.length%2){carry=bytes[bytes.length-1];bytes=bytes.slice(0,-1);}
|
||
if(!bytes.length) continue;
|
||
const copy=bytes.buffer.slice(bytes.byteOffset,bytes.byteOffset+bytes.byteLength);
|
||
session.bufferedFrames+=bytes.byteLength/2;
|
||
asset.started=true;
|
||
session.node.port.postMessage({type:'push',samples:copy,sampleRate:asset.sampleRate},[copy]);
|
||
}
|
||
}finally{
|
||
if((!active||token!==generation||session.cancelled)){try{await reader.cancel();}catch(_){ }}
|
||
session.controllers.delete(asset.controller);
|
||
}
|
||
}
|
||
|
||
async function drainPcmPlayback(session,token){
|
||
if(!session||!session.node||session.cancelled||!active||token!==generation) return;
|
||
if(session.playbackEnded) return;
|
||
session.node.port.postMessage({type:'end'});
|
||
await session.drained;
|
||
}
|
||
|
||
async function closePcmBeforeBlob(session,token){
|
||
if(!session||!session.node) return;
|
||
await drainPcmPlayback(session,token);
|
||
if(session.node){try{session.node.disconnect();}catch(_){ }session.node=null;}
|
||
if(session.context){try{session.context.close();}catch(_){ }session.context=null;}
|
||
session.gain=null;
|
||
session.drained=null;
|
||
indicator.classList.remove('is-playing');
|
||
}
|
||
|
||
async function playPrepared(asset,token){
|
||
if(asset.kind!=='pcm'){
|
||
if(playbackSession&&playbackSession.node) await closePcmBeforeBlob(playbackSession,token);
|
||
return playBlob(asset.blob,token);
|
||
}
|
||
try{
|
||
return await playPcm(asset,token);
|
||
}catch(error){
|
||
if(asset.started) throw error;
|
||
streamingCapability.tts=null;
|
||
if(playbackSession&&playbackSession.node) await closePcmBeforeBlob(playbackSession,token);
|
||
return playBlob(await fetchSpeech(asset.chunk,asset.language,asset.turnId,token),token);
|
||
}
|
||
}
|
||
|
||
function nextSpeechChunk(turn){
|
||
if(turn.cancelled) return Promise.resolve(null);
|
||
if(turn.queue.length) return Promise.resolve(turn.queue.shift());
|
||
if(turn.final) return Promise.resolve(null);
|
||
return new Promise(function(resolve){turn.waiters.push(resolve);});
|
||
}
|
||
|
||
function enqueueSpeech(turn,chunks){
|
||
chunks.forEach(function(chunk){
|
||
if(!chunk) return;
|
||
if(turn.waiters.length) turn.waiters.shift()(chunk); else turn.queue.push(chunk);
|
||
});
|
||
}
|
||
|
||
function finishSpeechQueue(turn){
|
||
turn.final=true;
|
||
while(turn.waiters.length&&!turn.queue.length) turn.waiters.shift()(null);
|
||
}
|
||
|
||
function scheduleSpeakingIdleFallback(turn,session){
|
||
// Interim-message turns speak in cycles (speak → tools → speak): once no
|
||
// further chunk is queued and the turn is not final, fall back to
|
||
// Thinking after the buffered audio has played out. The observer flips
|
||
// the state back to Speaking when the next segment yields a chunk.
|
||
cancelSpeakingIdleFallback(turn);
|
||
const bufferedMs=session&&session.sampleRate?Math.ceil(((session.bufferedFrames||0)/session.sampleRate)*1000):0;
|
||
turn.idleTimer=window.setTimeout(function(){
|
||
turn.idleTimer=null;
|
||
if(!active||turn.token!==generation||turn.cancelled||speechTurn!==turn) return;
|
||
if(turn.queue.length||turn.final) return;
|
||
if(state==='speaking') setState('thinking');
|
||
},bufferedMs+250);
|
||
}
|
||
|
||
function cancelSpeakingIdleFallback(turn){
|
||
if(turn&&turn.idleTimer){window.clearTimeout(turn.idleTimer);turn.idleTimer=null;}
|
||
}
|
||
|
||
async function runSpeechQueue(turn){
|
||
if(turn.running) return;
|
||
turn.running=true;
|
||
playbackSession={turnId:turn.turnId,controllers:new Set(),context:null,node:null,gain:null,sampleRate:0,bufferedFrames:0,drained:null,drainReject:null,playbackEnded:false,cancelled:false,lowWaterWake:null,drainWake:null,blobWake:null};
|
||
const session=playbackSession;
|
||
try{
|
||
let chunk=await nextSpeechChunk(turn);
|
||
let current=chunk?prepareSpeech(chunk,turn.language,turn.turnId,turn.token):null;
|
||
while(current){
|
||
const asset=await current;
|
||
// At most one later synthesis request exists while this asset plays.
|
||
// If streaming has not produced it yet, the waiter starts it the
|
||
// instant a punctuation-safe chunk arrives.
|
||
const next=nextSpeechChunk(turn);
|
||
const nextPrepared=next.then(function(value){
|
||
return value?prepareSpeech(value,turn.language,turn.turnId,turn.token):null;
|
||
});
|
||
// Barge-in can abort both the playing request and its one-ahead request.
|
||
// Observe the latter even when the cancelled current turn returns first.
|
||
nextPrepared.catch(function(){ });
|
||
turn.speakingChunk=asset.chunk||'';
|
||
await playPrepared(asset,turn.token);
|
||
turn.lastSpokenChunk=asset.chunk||turn.lastSpokenChunk;
|
||
turn.speakingChunk='';
|
||
if(!active||turn.token!==generation||turn.cancelled||session.cancelled) return;
|
||
if(!turn.queue.length&&!turn.final) scheduleSpeakingIdleFallback(turn,session);
|
||
current=await nextPrepared;
|
||
cancelSpeakingIdleFallback(turn);
|
||
if(current) setState('speaking');
|
||
}
|
||
await drainPcmPlayback(session,turn.token);
|
||
if(turn.final&&active&&turn.token===generation&&!turn.cancelled) restartSoon(turn.token,300);
|
||
}catch(error){
|
||
if(active&&turn.token===generation&&!turn.cancelled){
|
||
const message=errorMessage(error,'Local speech is unavailable');
|
||
setState('error',message);
|
||
toast(message);
|
||
restartSoon(turn.token,500);
|
||
}
|
||
}finally{
|
||
if(playbackSession===session) stopPlayback();
|
||
if(speechTurn===turn) speechTurn=null;
|
||
}
|
||
}
|
||
|
||
function collectAssistantResponse(){
|
||
const rows=assistantRows();
|
||
if(!rows.length) return {text:'',error:false};
|
||
const turn=assistantTurnOf(rows[rows.length-1]);
|
||
const response=readAssistantTurn(turn);
|
||
if(assistantBaseline){
|
||
if(rows.length<assistantBaseline.count) return {text:'',error:false};
|
||
if(turn===assistantBaseline.row&&response.text===assistantBaseline.text) return {text:'',error:response.error};
|
||
}
|
||
return response;
|
||
}
|
||
|
||
function currentAssistantText(){
|
||
return collectAssistantResponse().text;
|
||
}
|
||
|
||
function ensureSpeechTurn(token){
|
||
if(speechTurn) return speechTurn;
|
||
const sttDetected=takeSttLanguage(token);
|
||
speechTurn={
|
||
token:token,
|
||
turnId:thinkingTurnId||String(token)+'-'+String(++turnSequence),
|
||
language:sttDetected,
|
||
sttLanguage:sttDetected,
|
||
sourceText:'',
|
||
speakingChunk:'',
|
||
lastSpokenChunk:'',
|
||
idleTimer:null,
|
||
consumed:0,
|
||
first:true,
|
||
queue:[],
|
||
waiters:[],
|
||
final:false,
|
||
cancelled:false,
|
||
running:false,
|
||
};
|
||
return speechTurn;
|
||
}
|
||
|
||
function pumpAssistantResponse(token,isFinal){
|
||
if(!active||token!==generation||(state!=='thinking'&&state!=='speaking')) return;
|
||
if(isFinal){
|
||
// The completion callback marks a normally completed response: any
|
||
// interrupted-thought stitch from an earlier barge is now stale.
|
||
pendingStitch=null;
|
||
}
|
||
const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null;
|
||
if(thinkingSession&¤tSession&&thinkingSession!==currentSession){
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
clearSttLanguage();
|
||
stopResponseObserver();
|
||
cancelThinkingCues();
|
||
cancelSpeechTurn();
|
||
stopPlayback();
|
||
restartSoon(token,250);
|
||
return;
|
||
}
|
||
const response=collectAssistantResponse();
|
||
if(response.error){
|
||
handleAssistantResponseError(token);
|
||
return;
|
||
}
|
||
const text=response.text;
|
||
if(!text){
|
||
if(isFinal){
|
||
// First-sentence-stop guard: once a reply has begun speaking, the
|
||
// completion callback can fire a frame before the settle re-render
|
||
// re-stamps the answer onto data-raw-text, making this read transiently
|
||
// empty. Tearing down here would drop every unspoken sentence and jump
|
||
// to "Listening" after sentence one (even with the mic muted). While a
|
||
// reply is genuinely mid-flight (something was queued/spoken) retry for
|
||
// up to ~1s so the WHOLE reply is read, then close the queue cleanly so
|
||
// the buffered audio drains instead of a hard restart. A truly empty
|
||
// completion — no reply produced at all — still tears down immediately,
|
||
// so it never holds the speaking state open across the next utterance.
|
||
const midReply=!!(speechTurn&&(speechTurn.consumed>0||speechTurn.queue.length||(speechTurn.waiters&&speechTurn.waiters.length)));
|
||
if(midReply&&finalizeAttempts<8&&active&&token===generation&&(state==='thinking'||state==='speaking')){
|
||
finalizeAttempts+=1;
|
||
window.setTimeout(function(){pumpAssistantResponse(token,true);},120);
|
||
return;
|
||
}
|
||
finalizeAttempts=0;
|
||
if(midReply){
|
||
finishSpeechQueue(speechTurn);
|
||
stopResponseObserver();
|
||
thinkingSession=null;thinkingTurnId='';clearSttLanguage();cancelThinkingCues();
|
||
return;
|
||
}
|
||
thinkingSession=null;thinkingTurnId='';clearSttLanguage();stopResponseObserver();cancelThinkingCues();restartSoon(token,250);
|
||
}
|
||
return;
|
||
}
|
||
finalizeAttempts=0;
|
||
setConversationAssistantCaption(text);
|
||
// Any answer text, even a not-yet-speakable partial clause, owns the audio
|
||
// timeline from this point forward. A cue must never overlap or become part
|
||
// of the answer queue.
|
||
cancelThinkingCues();
|
||
const turn=ensureSpeechTurn(token);
|
||
if(turn.first){
|
||
// Reply-voice routing, finalized when the first chunk is cut: the
|
||
// turn's STT-detected language wins, but script evidence in the reply
|
||
// text corrects a wrong or missing detection (a Spanish reply must
|
||
// never be spoken by the English Amy voice), and a detection-less turn
|
||
// falls back to the reply-text heuristic. A correction on a
|
||
// detection-less turn also re-biases the sticky session language so
|
||
// the next streaming STT turn recognizes the switch promptly.
|
||
// A user-forced language (FIX 3) wins over every auto signal: the reply is
|
||
// spoken by the chosen voice regardless of script evidence or detection.
|
||
const resolved=forcedLanguage||strongReplyLanguage(text)||turn.sttLanguage||detectReplyLanguage(text);
|
||
if(resolved) turn.language=resolved;
|
||
if(!forcedLanguage&&!turn.sttLanguage&&resolved) sessionLanguage=resolved;
|
||
}
|
||
if(text.length<turn.sourceText.length||!text.startsWith(turn.sourceText)){
|
||
// Renderers can revise the still-unspoken tail. Already-spoken text is
|
||
// immutable. On the completion callback, advance from the old spoken
|
||
// offset and close the queue even when the renderer rewrote an earlier
|
||
// span; replaying the revised prefix would be more disruptive than
|
||
// preserving the already-heard words.
|
||
if(turn.consumed>0){
|
||
if(!isFinal) return;
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
stopResponseObserver();
|
||
const revisedTail=text.slice(Math.min(turn.consumed,text.length)).trim();
|
||
if(revisedTail){
|
||
const revised=adaptiveChunks(revisedTail,true,false);
|
||
enqueueSpeech(turn,revised.chunks.length?revised.chunks:[revisedTail]);
|
||
turn.consumed=text.length;
|
||
}
|
||
finishSpeechQueue(turn);
|
||
return;
|
||
}
|
||
turn.sourceText='';
|
||
}
|
||
turn.sourceText=text;
|
||
const remaining=text.slice(turn.consumed).trimStart();
|
||
const skipped=text.slice(turn.consumed).length-remaining.length;
|
||
const extracted=adaptiveChunks(remaining,!!isFinal,turn.first);
|
||
if(extracted.chunks.length){
|
||
turn.first=false;
|
||
turn.consumed+=skipped+extracted.consumed;
|
||
enqueueSpeech(turn,extracted.chunks);
|
||
setState('speaking');
|
||
runSpeechQueue(turn);
|
||
}
|
||
if(isFinal){
|
||
thinkingSession=null;
|
||
thinkingTurnId='';
|
||
stopResponseObserver();
|
||
const tail=text.slice(turn.consumed).trim();
|
||
if(tail){enqueueSpeech(turn,[tail]);turn.consumed=text.length;}
|
||
finishSpeechQueue(turn);
|
||
}
|
||
}
|
||
|
||
function startResponseObserver(token){
|
||
stopResponseObserver();
|
||
// A bounded poll is deliberately used in addition to the upstream
|
||
// completion callback. It sees data-raw-text while SSE is still appending,
|
||
// so the first complete sentence can reach Piper before generation ends.
|
||
responsePollTimer=window.setInterval(function(){pumpAssistantResponse(token,false);},75);
|
||
}
|
||
|
||
function speakResponse(token){
|
||
pumpAssistantResponse(token,true);
|
||
}
|
||
|
||
function validStreamingCapability(payload){
|
||
if(!payload||typeof payload!=='object') return {tts:null,stt:null,preflight:null};
|
||
const tts=payload.tts&&payload.tts.available===true&&payload.tts.transport==='http'&&payload.tts.format==='pcm_s16le'?payload.tts:null;
|
||
const stt=payload.stt&&payload.stt.available===true&&payload.stt.transport==='websocket'&&payload.stt.format==='pcm_s16le'&&payload.stt.sample_rate===16000?payload.stt:null;
|
||
const preflight=payload.preflight&&payload.preflight.available===true&&payload.preflight.path===VOICE_PREFLIGHT_URL&&payload.preflight.advisory===true?payload.preflight:null;
|
||
return {tts:tts,stt:stt,preflight:preflight};
|
||
}
|
||
|
||
async function discoverStreamingCapability(){
|
||
try{
|
||
const response=await fetch(STREAMING_CAPABILITY_URL,{cache:'no-store'});
|
||
if(!response.ok) return;
|
||
streamingCapability=validStreamingCapability(await response.json());
|
||
}catch(_){
|
||
streamingCapability={tts:null,stt:null,preflight:null};
|
||
}
|
||
}
|
||
|
||
function activate(){
|
||
generation+=1;
|
||
const token=generation;
|
||
active=true;
|
||
clearErrorTimer();
|
||
clearSttLanguage();
|
||
sessionLanguage='';
|
||
forcedLanguage='';
|
||
finalizeAttempts=0;
|
||
modeBtn.classList.add('active');
|
||
toast('Hands-free private voice mode on');
|
||
openConversationOverlay();
|
||
if(typeof window.stopTTS==='function') window.stopTTS();
|
||
if(typeof S!=='undefined'&&S.busy){setState('thinking');return;}
|
||
startListening(token);
|
||
}
|
||
|
||
function onVoiceClick(event){
|
||
if(!ready) return;
|
||
event.preventDefault();
|
||
event.stopImmediatePropagation();
|
||
if(active) deactivate(true); else activate();
|
||
}
|
||
|
||
async function initialize(){
|
||
try{
|
||
const response=await fetch('/api/transcribe/capability',{cache:'no-store'});
|
||
const capability=await response.json().catch(function(){return {};});
|
||
if(!response.ok||!capability.available||capability.provider!=='local_command') return;
|
||
ready=true;
|
||
discoverStreamingCapability();
|
||
if(localStorage.getItem('hermes-atlas-voice-initialized')!=='1'){
|
||
localStorage.setItem('hermes-atlas-voice-initialized','1');
|
||
localStorage.setItem('hermes-voice-mode-button','true');
|
||
localStorage.setItem('hermes-tts-engine','atlas');
|
||
localStorage.setItem('hermes-tts-enabled','true');
|
||
}
|
||
// Move existing installations to the lower-latency silence window once;
|
||
// subsequent user changes remain untouched.
|
||
if(localStorage.getItem('hermes-atlas-voice-latency-v2')!=='1'){
|
||
localStorage.setItem('hermes-atlas-voice-latency-v2','1');
|
||
localStorage.setItem('hermes-voice-silence-ms','1100');
|
||
}
|
||
// Seed the hands-free speech-rate default once; later user changes to
|
||
// hermes-voice-tts-speed (clamped to 0.5-2.0) are respected as-is.
|
||
if(localStorage.getItem('hermes-voice-tts-speed')===null){
|
||
localStorage.setItem('hermes-voice-tts-speed',String(TTS_SPEED_DEFAULT));
|
||
}
|
||
const selector=document.getElementById('settingsTtsEngine');
|
||
if(selector&&!selector.querySelector('option[value="atlas"]')){
|
||
const option=document.createElement('option');
|
||
option.value='atlas';
|
||
option.textContent='Atlas Jetson (private)';
|
||
selector.insertBefore(option,selector.firstChild);
|
||
}
|
||
modeBtn.style.display=localStorage.getItem('hermes-voice-mode-button')==='false'?'none':'';
|
||
modeBtn.addEventListener('click',onVoiceClick,true);
|
||
window._applyVoiceModePref=function(){
|
||
if(typeof originalApplyPreference==='function') originalApplyPreference();
|
||
if(ready){
|
||
const enabled=localStorage.getItem('hermes-voice-mode-button')!=='false';
|
||
modeBtn.style.display=enabled?'':'none';
|
||
if(!enabled&&active) deactivate(false);
|
||
}
|
||
};
|
||
window.autoReadLastAssistant=function(){
|
||
if(active){
|
||
if(suppressAutoRead||bargeCancelPromise||state==='listening'||state==='transcribing') return;
|
||
if(state==='thinking'||state==='speaking'){speakResponse(generation);return;}
|
||
}
|
||
if(typeof originalAutoRead==='function') originalAutoRead.apply(this,arguments);
|
||
};
|
||
window._voiceModeActive=function(){return active;};
|
||
window._voiceModeDeactivate=function(){deactivate(false);};
|
||
window._voiceModeImmediateSend=function(){
|
||
if(active&&recorder&&recorder.state!=='inactive') recorder.stop();
|
||
};
|
||
}catch(_){
|
||
// The upstream browser voice implementation remains available as fallback.
|
||
}
|
||
}
|
||
|
||
// Behaviour-neutral instrumentation surface: pure references to the internal
|
||
// extraction and one-ahead chunking helpers so a deterministic node probe can
|
||
// drive them against a realistic rendered assistant turn (never touched by the
|
||
// running app). Regression-locks the caption/TTS extraction contract.
|
||
if(typeof window!=='undefined'){
|
||
window.__atlasVoiceInternals={
|
||
readAssistantTurn:readAssistantTurn,
|
||
collectAssistantResponse:collectAssistantResponse,
|
||
currentAssistantText:currentAssistantText,
|
||
adaptiveChunks:adaptiveChunks,
|
||
sentenceEnd:sentenceEnd,
|
||
cleanForSpeech:cleanForSpeech,
|
||
};
|
||
}
|
||
|
||
initialize();
|
||
})();
|