// Deterministic continuous-capture probe for dockerfiles/hermes-webui-atlas-voice.js. // // Drives the real script through complete streaming hands-free turns against a // stub STT WebSocket whose VAD, speculative EOS-freeze, resume and commit // semantics faithfully mirror dockerfiles/hermes-jetson-stt-server.py. Words // are encoded as distinct constant PCM amplitudes so the "transcript" of any // audio snapshot is exactly the ordered set of words whose samples survived // endpointing — a first-word-only regression is directly observable as a // truncated transcript in the sent composer text. // // node hermes_voice_capture_probe.js // // Prints one JSON object per scenario to stdout; the python wrapper asserts. 'use strict'; const fs = require('fs'); const vm = require('vm'); const SCRIPT_PATH = process.argv[2]; if (!SCRIPT_PATH) { throw new Error('usage: hermes_voice_capture_probe.js '); } const SOURCE = fs.readFileSync(SCRIPT_PATH, 'utf8'); class FakeURL extends URL {} FakeURL.createObjectURL = () => 'blob:capture-probe'; FakeURL.revokeObjectURL = () => {}; const SAMPLE_RATE = 16000; const TICK_MS = 100; const TICK_SAMPLES = (SAMPLE_RATE * TICK_MS) / 1000; // Server constants mirrored from hermes-jetson-stt-server.py. const SERVER_END_SILENCE_SAMPLES = (SAMPLE_RATE * 650) / 1000; const SERVER_TAIL_SAMPLES = (SAMPLE_RATE * 220) / 1000; function flush() { return new Promise((resolve) => { let hops = 0; (function hop() { hops += 1; if (hops > 16) { resolve(); return; } setImmediate(hop); })(); }); } function makeElement(id) { const element = { id, value: '', textContent: '', className: '', dataset: {}, attributes: {}, children: [], style: { values: new Map(), display: '', setProperty(name, value) { this.values.set(name, String(value)); }, getPropertyValue(name) { return this.values.get(name) || ''; }, removeProperty(name) { this.values.delete(name); }, }, listeners: [], setAttribute(name, value) { this.attributes[name] = String(value); }, getAttribute(name) { return Object.prototype.hasOwnProperty.call(this.attributes, name) ? this.attributes[name] : null; }, addEventListener(type, handler) { this.listeners.push({ type, handler }); }, removeEventListener(type, handler) { element.listeners = element.listeners.filter((entry) => entry.handler !== handler); }, appendChild(child) { element.children.push(child); child.parentNode = element; return child; }, removeChild(child) { element.children = element.children.filter((entry) => entry !== child); child.parentNode = null; return child; }, contains() { return false; }, focus() {}, querySelector() { return null; }, querySelectorAll() { return []; }, insertBefore() {}, click() { const event = { preventDefault() {}, stopImmediatePropagation() {} }; element.listeners .filter((entry) => entry.type === 'click') .forEach((entry) => entry.handler(event)); }, }; element.classList = { add(name) { const names = new Set(element.className.split(/\s+/).filter(Boolean)); names.add(name); element.className = [...names].join(' '); }, remove(name) { element.className = element.className .split(/\s+/).filter((value) => value && value !== name).join(' '); }, contains(name) { return element.className.split(/\s+/).includes(name); }, toggle(name, force) { const present = this.contains(name); const next = force === undefined ? !present : !!force; if (next) this.add(name); else this.remove(name); return next; }, }; return element; } // ── Faithful assistant-turn DOM ───────────────────────────────────────────── // A minimal but real querySelectorAll / closest / getAttribute / hidden / // recursive-textContent implementation, so the interim-acknowledgement FOLD can // be reproduced exactly as the live renderer performs it: the interim segment is // re-tagged .assistant-segment-worklog-source + hidden + aria-hidden, at which // point the script's real collectAssistantResponse() returns '' for it (the same // extraction the response probe locks) and the final answer renders as its own // new .assistant-segment. Supports the compound selectors the extraction uses. function parseDomSelector(selector) { return String(selector).split(',').map((group) => { const term = group.trim(); const parts = []; const re = /([.#]?[\w-]+)|\[([\w-]+)(?:([~|^$*]?=)"?([^"\]]*)"?)?\]/g; let m; while ((m = re.exec(term))) { if (m[1]) { if (m[1][0] === '.') parts.push({ kind: 'class', value: m[1].slice(1) }); else parts.push({ kind: 'tag', value: m[1].toLowerCase() }); } else if (m[2]) { parts.push({ kind: 'attr', name: m[2], op: m[3] || null, value: m[4] }); } } return parts; }).filter((parts) => parts.length); } class DomNode { constructor(tag) { this.tag = String(tag || 'div').toLowerCase(); this.className = ''; this.attributes = new Map(); this.children = []; this.parentNode = null; this.hidden = false; this._text = ''; const self = this; this.dataset = new Proxy({}, { get(_t, key) { if (typeof key !== 'string') return undefined; const attr = 'data-' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()); return self.attributes.has(attr) ? self.attributes.get(attr) : undefined; }, has(_t, key) { const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()); return self.attributes.has(attr); }, }); } get classList() { const el = this; return { add(name) { const s = new Set(el.className.split(/\s+/).filter(Boolean)); s.add(name); el.className = [...s].join(' '); }, remove(name) { el.className = el.className.split(/\s+/).filter((v) => v && v !== name).join(' '); }, contains(name) { return el.className.split(/\s+/).indexOf(name) >= 0; }, }; } setAttribute(name, value) { this.attributes.set(name, String(value)); if (name === 'class') this.className = String(value); if (name === 'hidden') this.hidden = true; } getAttribute(name) { if (name === 'class') return this.className || null; return this.attributes.has(name) ? this.attributes.get(name) : null; } appendChild(child) { child.parentNode = this; this.children.push(child); return child; } set textContent(value) { this._text = String(value); this.children = []; } get textContent() { if (this.children.length) return this.children.map((c) => c.textContent).join(''); return this._text; } _matchesTerm(parts) { return parts.every((p) => { if (p.kind === 'class') return this.classList.contains(p.value); if (p.kind === 'tag') return this.tag === p.value; if (p.kind === 'attr') { if (!this.attributes.has(p.name)) return false; return p.op ? this.attributes.get(p.name) === p.value : true; } return false; }); } matches(selector) { return parseDomSelector(selector).some((parts) => this._matchesTerm(parts)); } _walk(out) { for (const c of this.children) { out.push(c); c._walk(out); } return out; } querySelectorAll(selector) { const groups = parseDomSelector(selector); return this._walk([]).filter((node) => groups.some((parts) => node._matchesTerm(parts))); } querySelector(selector) { const all = this.querySelectorAll(selector); return all.length ? all[0] : null; } closest(selector) { const groups = parseDomSelector(selector); let node = this; while (node) { if (groups.some((parts) => node._matchesTerm(parts))) return node; node = node.parentNode; } return null; } } function domEl(tag, className, attrs, text) { const node = new DomNode(tag); if (className) node.setAttribute('class', className); if (attrs) Object.keys(attrs).forEach((k) => node.setAttribute(k, attrs[k])); if (text !== undefined) node.textContent = text; return node; } // A rendered assistant turn: role header, worklog chip, and an assistant-turn // blocks container the interim/final segments hang off of. function buildAssistantTurn() { const turn = domEl('div', 'msg-row assistant-turn', { 'data-role': 'assistant', 'data-session-id': 'session-1' }); const role = domEl('div', 'msg-role assistant'); role.appendChild(domEl('div', 'role-icon assistant', null, 'H')); role.appendChild(domEl('span', 'msg-role-name', null, 'Hermes')); turn.appendChild(role); const blocks = domEl('div', 'assistant-turn-blocks'); turn.appendChild(blocks); turn.blocks = blocks; return turn; } // A live (still-streaming) answer segment: no data-raw-text yet, so the script // reads its answer .msg-body — exactly the interim acknowledgement path. function addLiveAnswerSegment(turn, text) { const seg = domEl('div', 'assistant-segment', { 'data-live-assistant': '1' }); seg.appendChild(domEl('div', 'msg-body', null, text)); turn.blocks.appendChild(seg); return seg; } // A settled answer segment carrying the renderer-stamped clean answer text. function addSettledAnswerSegment(turn, text) { const seg = domEl('div', 'assistant-segment', { 'data-raw-text': text }); seg.appendChild(domEl('div', 'msg-body', null, text)); turn.blocks.appendChild(seg); return seg; } // Fold an interim segment into the hidden worklog source, exactly as the live // renderer does when a tool call begins: the extraction now excludes it. function foldSegmentIntoWorklog(seg) { seg.setAttribute('class', 'assistant-segment assistant-segment-worklog-source'); seg.setAttribute('aria-hidden', 'true'); seg.hidden = true; } // Faithful port of StreamingTranscription's endpointing/speculation contract. class StubSttServer { constructor(words) { // words: {amplitude(int16) -> word string} this.words = words; this.chunks = []; // Int16Array chunks in arrival order this.totalSamples = 0; this.noiseFloor = 0.004; this.heardSpeech = false; this.atEos = false; this.silenceSamples = 0; this.lastSpeechSample = 0; this.clientActive = false; this.epoch = 0; this.frozenSnapshotSamples = -1; this.committed = false; this.events = []; } snapshotSamples() { if (this.atEos && this.frozenSnapshotSamples >= 0) return this.frozenSnapshotSamples; let cutoff = this.totalSamples; if (this.heardSpeech && this.lastSpeechSample) { cutoff = Math.min(cutoff, this.lastSpeechSample + SERVER_TAIL_SAMPLES); } return cutoff; } append(int16) { if (this.committed) throw new Error('server: audio after commit'); this.chunks.push(int16); this.totalSamples += int16.length; let energy = 0; for (let index = 0; index < int16.length; index += 1) { const value = int16[index] / 32768; energy += value * value; } const rms = int16.length ? Math.sqrt(energy / int16.length) : 0; const threshold = Math.max(0.012, this.noiseFloor * 2.5 + 0.004); const speech = this.clientActive || rms >= threshold; if (!this.heardSpeech && !speech) { this.noiseFloor = this.noiseFloor * 0.96 + rms * 0.04; } if (speech) { if (this.atEos) { this.epoch += 1; this.frozenSnapshotSamples = -1; this.events.push('recover-by-rms'); } this.heardSpeech = true; this.atEos = false; this.silenceSamples = 0; this.lastSpeechSample = this.totalSamples; } else if (this.heardSpeech) { this.silenceSamples += int16.length; if (this.silenceSamples >= SERVER_END_SILENCE_SAMPLES && !this.atEos) { this.atEos = true; this.frozenSnapshotSamples = this.snapshotSamples(); this.events.push('auto-eos-freeze'); } } } speculate() { this.clientActive = false; this.atEos = true; this.frozenSnapshotSamples = this.snapshotSamples(); this.events.push('speculate-freeze'); } resume() { this.atEos = false; this.silenceSamples = 0; this.epoch += 1; this.clientActive = true; this.frozenSnapshotSamples = -1; this.events.push('resume-clear'); } transcript() { const cutoff = this.snapshotSamples(); const decoded = []; let position = 0; let currentWord = null; let runLength = 0; const flushRun = () => { // Ignore sub-40ms runs: resampler transition samples, not words. if (currentWord && runLength >= SAMPLE_RATE * 0.04) decoded.push(currentWord); currentWord = null; runLength = 0; }; for (const chunk of this.chunks) { for (let index = 0; index < chunk.length; index += 1) { if (position >= cutoff) { flushRun(); return decoded.join(' '); } position += 1; const magnitude = Math.abs(chunk[index]); let word = null; for (const [amplitude, name] of Object.entries(this.words)) { if (Math.abs(magnitude - Number(amplitude)) <= 200) { word = name; break; } } if (word === currentWord) { runLength += 1; } else { flushRun(); currentWord = word; runLength = word ? 1 : 0; } } } flushRun(); return decoded.join(' '); } } function makeHarness(options = {}) { const clock = { now: 1000000 }; const timeouts = []; const intervals = new Map(); let timerId = 1; const sends = []; const toasts = []; const transcribeUploads = []; const ttsCalls = []; const servers = []; const captureNodes = []; const analysers = []; const recorders = []; let assistantRows = []; let streamCounter = 0; const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' }; const micTracks = []; const sinkCalls = []; const elements = {}; ['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg', 'emptyState'] .forEach((id) => { elements[id] = makeElement(id); }); const bodyElement = makeElement('body'); // ── Microphone model ──────────────────────────────────────────────────── // Per tick the mic has one (analyser level, pcm amplitude) pair. The level // feeds the script's VAD analyser; the amplitude becomes the PCM the // capture worklet delivers to the streaming session. const mic = { level: 0, amplitude: 0 }; class FakeAnalyser { constructor() { this.fftSize = 0; analysers.push(this); } getByteTimeDomainData(samples) { for (let index = 0; index < samples.length; index += 1) { const offset = Math.round(mic.level * 127); samples[index] = 128 + (index % 2 ? offset : -offset); } } disconnect() {} } class FakeAudioContext { constructor() { this.sampleRate = SAMPLE_RATE; this.destination = {}; this.audioWorklet = { addModule: async () => {} }; this.closed = false; } createAnalyser() { return new FakeAnalyser(); } createBiquadFilter() { return { type: '', frequency: { value: 0 }, Q: { value: 0 }, connect() {}, disconnect() {} }; } createMediaStreamSource() { return { connect() {}, disconnect() {} }; } createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; } resume() { return Promise.resolve(); } setSinkId(id) { sinkCalls.push({ kind: 'context', sink: id }); return Promise.resolve(); } close() { this.closed = true; return Promise.resolve(); } } class FakeAudioWorkletNode { constructor(_context, name) { this.name = name; this.connected = true; const node = this; this.port = { onmessage: null, postMessage(message) { if ((message || {}).type === 'flush') { queueMicrotask(() => { if (node.port.onmessage) node.port.onmessage({ data: { type: 'flushed' } }); }); } }, }; if (name === 'atlas-pcm-capture') captureNodes.push(this); } connect() {} disconnect() { this.connected = false; } } class FakeMediaRecorder { static isTypeSupported() { return true; } constructor(stream, recorderOptions) { this.stream = stream; this.state = 'inactive'; this.mimeType = (recorderOptions && recorderOptions.mimeType) || 'audio/webm;codecs=opus'; this.ondataavailable = null; this.onstop = null; recorders.push(this); } start() { this.state = 'recording'; } stop() { if (this.state === 'inactive') return; this.state = 'inactive'; if (this.ondataavailable) { this.ondataavailable({ data: { size: 512, type: 'audio/webm;codecs=opus' } }); } const recorder = this; // Real recorders finalize asynchronously. queueMicrotask(() => { if (recorder.onstop) recorder.onstop(); }); } } class FakeWebSocket { constructor(url, protocols) { this.url = url; this.protocols = protocols; this.readyState = 0; this.bufferedAmount = 0; this.binaryType = ''; this.onopen = null; this.onmessage = null; this.onerror = null; this.onclose = null; this.server = new StubSttServer(words); this.turnId = ''; servers.push(this); const socket = this; queueMicrotask(() => { if (socket.readyState !== 0) return; socket.readyState = 1; if (socket.onopen) socket.onopen(); }); } deliver(payload) { const socket = this; queueMicrotask(() => { if (socket.readyState !== 1) return; if (socket.onmessage) socket.onmessage({ data: JSON.stringify(payload) }); }); } send(data) { if (this.readyState !== 1) throw new Error('socket not open'); if (typeof data === 'string') { const message = JSON.parse(data); if (message.type === 'start') { this.turnId = message.turn_id; this.startLanguage = message.language; return; } if (message.type === 'speculate') { this.server.speculate(); return; } if (message.type === 'resume') { this.server.resume(); return; } if (message.type === 'cancel') { this.server.events.push('cancel'); return; } if (message.type === 'commit') { this.server.committed = true; this.deliver({ type: 'final', turn_id: this.turnId, transcript: this.server.transcript(), language: 'en', }); } return; } this.server.append(new Int16Array(data)); // Mirror the server's rolling-partial stream: one stable partial per // decoded-transcript change, so the client's dynamic endpoint sees the // same signal the Jetson emits. const partialText = this.server.transcript(); if (partialText && partialText !== this.lastPartialText) { this.lastPartialText = partialText; this.partialRevision = (this.partialRevision || 0) + 1; this.deliver({ type: 'partial', rolling: true, turn_id: this.turnId, revision: this.partialRevision, transcript: partialText, stable_transcript: partialText, }); } } close(code) { this.readyState = 3; this.closeCode = code; } } async function fetchStub(url, init) { if (url === '/api/transcribe/capability') { return { ok: true, status: 200, json: async () => ({ available: true, provider: 'local_command' }) }; } if (url === '/api/voice/streaming/capability') { return { ok: true, status: 200, json: async () => ({ tts: { available: false }, stt: { available: true, transport: 'websocket', format: 'pcm_s16le', sample_rate: 16000 }, preflight: { available: false }, }), }; } if (url === '/api/transcribe') { transcribeUploads.push({ body: init && init.body }); return { ok: true, status: 200, json: async () => ({ transcript: 'CONTAINER-FALLBACK', language: 'en' }) }; } if (url === '/api/tts') { ttsCalls.push(JSON.parse((init && init.body) || '{}')); return { ok: true, status: 200, blob: async () => ({ synthetic: true }), json: async () => ({}) }; } if (String(url).indexOf('api/chat/cancel') >= 0) { context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; return { ok: true, status: 200, json: async () => ({ cancelled: true }) }; } throw new Error(`unexpected fetch: ${url}`); } const storage = new Map(); const context = { AbortController, console, Uint8Array, Int16Array, Float32Array, ArrayBuffer, DataView, Promise, Math, JSON, String, Number, Error, parseInt, parseFloat, isNaN, Set, Map, Array, Object, URL: FakeURL, queueMicrotask, Date: { now: () => clock.now }, Blob: function Blob(parts, blobOptions) { this.parts = parts; this.type = (blobOptions || {}).type || ''; this.size = 512; }, File: function File(parts, name, fileOptions) { this.parts = parts; this.name = name; this.type = (fileOptions || {}).type || ''; }, FormData: function FormData() { this.entries = []; this.append = (key, value) => this.entries.push([key, value]); }, Audio: function Audio() { this.play = () => Promise.resolve(); this.pause = () => {}; this.setSinkId = (id) => { sinkCalls.push({ kind: 'audio', sink: id }); return Promise.resolve(); }; this.onended = null; this.onerror = null; }, WebSocket: FakeWebSocket, AudioWorkletNode: FakeAudioWorkletNode, MediaRecorder: FakeMediaRecorder, AudioContext: FakeAudioContext, fetch: fetchStub, localStorage: { getItem: (key) => (storage.has(key) ? storage.get(key) : null), setItem: (key, value) => { storage.set(key, String(value)); }, removeItem: (key) => { storage.delete(key); }, }, navigator: { mediaDevices: { getSupportedConstraints: () => ({}), ...(options.outputs ? { enumerateDevices: async () => options.outputs } : {}), getUserMedia: async () => { const track = { stop() {}, enabled: true, getSettings: () => ({ echoCancellation: true }), }; micTracks.push(track); return { getTracks: () => [track], getAudioTracks: () => [track] }; }, }, }, document: { baseURI: 'https://chat.test/', body: bodyElement, getElementById: (id) => elements[id] || null, querySelectorAll: () => assistantRows, createElement: (tag) => makeElement(`created-${tag}`), addEventListener() {}, removeEventListener() {}, }, location: { protocol: 'https:', host: 'chat.test', href: 'https://chat.test/' }, crypto: { getRandomValues(bytes) { for (let i = 0; i < bytes.length; i += 1) bytes[i] = i + 1; } }, S: { session: { session_id: 'session-1' }, busy: false, activeStreamId: null }, setTimeout: (fn, delay) => { const id = timerId; timerId += 1; timeouts.push({ id, fn, at: clock.now + (delay || 0) }); return id; }, clearTimeout: (id) => { const index = timeouts.findIndex((entry) => entry.id === id); if (index >= 0) timeouts.splice(index, 1); }, setInterval: (fn, delay) => { const id = timerId; timerId += 1; intervals.set(id, { fn, delay: delay || 0 }); return id; }, clearInterval: (id) => { intervals.delete(id); }, }; context.window = context; context.window.location = context.location; context.window.crypto = context.crypto; context.showToast = (message) => { toasts.push(message); }; context.send = () => { streamCounter += 1; sends.push(elements.msg.value); context.S.busy = true; context.S.activeStreamId = `stream-${streamCounter}`; if (context.S.session) context.S.session.active_stream_id = context.S.activeStreamId; }; context.autoResize = () => {}; context.stopTTS = () => {}; context._stripForTTS = (text) => text; vm.createContext(context); vm.runInContext(SOURCE, context, { filename: 'atlas-voice.js' }); function runDueTimeouts() { const due = timeouts.filter((entry) => entry.at <= clock.now); due.forEach((entry) => { const index = timeouts.indexOf(entry); if (index >= 0) timeouts.splice(index, 1); entry.fn(); }); } function currentCaptureNode() { for (let index = captureNodes.length - 1; index >= 0; index -= 1) { if (captureNodes[index].connected) return captureNodes[index]; } return null; } function deliverMicFrame() { const node = currentCaptureNode(); if (!node || !node.port.onmessage) return; const samples = new Float32Array(TICK_SAMPLES); const value = mic.amplitude / 32768; for (let index = 0; index < samples.length; index += 1) { samples[index] = index % 2 ? value : -value; } node.port.onmessage({ data: { type: 'pcm', samples: samples.buffer } }); } async function tick(ms) { const steps = Math.max(1, Math.round(ms / TICK_MS)); for (let step = 0; step < steps; step += 1) { clock.now += TICK_MS; deliverMicFrame(); Array.from(intervals.values()).forEach((entry) => entry.fn()); runDueTimeouts(); // eslint-disable-next-line no-await-in-loop await flush(); } } async function speak(word, ms) { const amplitude = Number(Object.keys(words).find((key) => words[key] === word)); if (!amplitude) throw new Error(`unknown word: ${word}`); mic.level = Math.min(0.9, amplitude / 32768 + 0.25); mic.amplitude = amplitude; await tick(ms); } async function silence(ms) { mic.level = 0; mic.amplitude = 0; await tick(ms); } return { context, elements, sends, toasts, transcribeUploads, ttsCalls, servers, recorders, clock, tick, speak, silence, flush, runDueTimeouts, setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; }, setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; }, // A TRANSIENT provider error turn: an error envelope carrying a // .provider-error-details block (so the extraction reports error), plus the // last-assistant "regenerate" action button the auto-retry clicks. The // button records each click so the scenario can assert the bounded retry. setAssistantTransientError(text) { // An errored turn is a COMPLETED turn: its stream ended, so the session is // no longer busy (the same precondition the app's regenerate action needs). context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; const turn = buildAssistantTurn(); const seg = domEl('div', 'assistant-segment'); seg.appendChild(domEl('div', 'msg-body', null, text)); seg.appendChild(domEl('details', 'provider-error-details', null, text)); turn.blocks.appendChild(seg); const foot = domEl('div', 'msg-foot'); const actions = domEl('span', 'msg-actions'); const regen = domEl('button', 'msg-action-btn', { onclick: 'regenerateResponse(this)', title: 'regenerate' }); regen.click = () => { // Mirror the app's regenerate: truncate the errored turn and start a // fresh stream. The scenario injects the next state (another error, or a // recovered answer) to model whether the retried turn succeeds. this._regenerateClicks = (this._regenerateClicks || 0) + 1; assistantRows = []; context.S.busy = true; streamCounter += 1; context.S.activeStreamId = `stream-${streamCounter}`; }; actions.appendChild(regen); foot.appendChild(actions); turn.appendChild(foot); assistantRows = [turn]; }, regenerateClicks() { return this._regenerateClicks || 0; }, endStream() { context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; }, clearAssistantRows() { assistantRows = []; }, // ── Interim-fold reproduction ───────────────────────────────────────── // Stream an interim acknowledgement as a live answer segment, then FOLD it // into the hidden worklog source (as the renderer does at the tool call) and // render the final answer as its own new segment. setInterimAck(text) { const turn = buildAssistantTurn(); this._foldTurn = turn; this._interimSeg = addLiveAnswerSegment(turn, text); assistantRows = [turn]; }, foldInterimAck() { if (this._interimSeg) foldSegmentIntoWorklog(this._interimSeg); }, setFinalAnswer(text) { if (!this._foldTurn) return; addSettledAnswerSegment(this._foldTurn, text); }, speechTurnSnapshot() { const internals = context.window.__atlasVoiceInternals; return internals && typeof internals.speechTurnSnapshot === 'function' ? internals.speechTurnSnapshot() : null; }, ttsTexts() { return ttsCalls.map((request) => request.text); }, sinkCalls() { return sinkCalls.slice(); }, state() { return elements.voiceModeBar.dataset.voiceState || ''; }, body() { return bodyElement; }, trackStates() { return micTracks.map((track) => track.enabled); }, overlay() { return bodyElement.children.find( (child) => String(child.className).indexOf('voice-conversation') >= 0, ) || null; }, completeResponse() { context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; context.autoReadLastAssistant(); }, async start() { await flush(); elements.btnVoiceMode.click(); await flush(); await flush(); }, // Boot the script (runs initialize(), which injects the empty-state // conversation button when voice is available) WITHOUT entering voice mode. async initVoice() { await flush(); await flush(); }, emptyConversationButton() { const empty = elements.emptyState; return (empty.children || []).find( (child) => child && child.id === 'btnEmptyConversation', ) || null; }, }; } const scenarios = {}; // The reported live regression: a normal multi-word utterance with a natural // inter-word pause long enough for the client to speculate (>=605ms at the // default 1100ms endpoint) must still transcribe completely. scenarios.multiword_utterance_with_interword_pause = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 400); await harness.silence(700); // client speculates at ~605ms await harness.speak('bravo', 600); await harness.silence(1400); // past the 1100ms endpoint await harness.tick(400); // commit retries + final delivery return { sends: harness.sends, state: harness.state(), serverEvents: harness.servers.map((socket) => socket.server.events), toasts: harness.toasts, }; }; // Continuous capture: the mic stays hot after an utterance is dispatched, and // the next utterance (spoken while the first response is still in flight) // barges in and is transcribed completely, including words after a pause. scenarios.second_utterance_during_response_is_complete = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 500); await harness.silence(1400); await harness.tick(400); // first send dispatched const firstSends = harness.sends.slice(); // The model is now answering (S.busy). Assistant text starts streaming. harness.setAssistantReply('Partial answer already visible.'); await harness.tick(200); // User talks over the response with a two-word interruption + pause. await harness.speak('bravo', 500); await harness.silence(700); await harness.speak('charlie', 500); await harness.silence(1400); await harness.tick(400); return { firstSends, sends: harness.sends, state: harness.state(), serverEvents: harness.servers.map((socket) => socket.server.events), toasts: harness.toasts, }; }; // The mic never re-acquires getUserMedia across utterances and a second // utterance right after a normally completed response is sent alone: a // normal completion clears any stitch context. scenarios.back_to_back_utterances_stay_hot = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); // first send dispatched (past the endpoint) const sendsAfterFirst = harness.sends.slice(); harness.completeResponse(); await harness.tick(400); await harness.speak('bravo', 1300); await harness.silence(1500); await harness.tick(400); return { sendsAfterFirst, sends: harness.sends, micAcquisitions: harness.trackStates().length, recorderCount: harness.recorders.length, serverCount: harness.servers.length, toasts: harness.toasts, }; }; // The exact live regression: "The … rest of the sentence" // used to endpoint on the first inter-word gap (>=1100ms) and send the first // word alone. A young utterance now holds its endpoint to 1800ms, so the // pause stays inside one utterance and one complete message is sent. scenarios.thinking_pause_after_first_word_does_not_split = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 400); await harness.silence(1300); // longer than the old 1100ms endpoint await harness.speak('bravo', 700); await harness.silence(2000); await harness.tick(400); return { sends: harness.sends, serverEvents: harness.servers.map((socket) => socket.server.events), toasts: harness.toasts, }; }; // Conversation mode: activation builds the full-screen overlay, captions // follow the turn, mute drives the real capture track, and both the exit // control and Escape tear the overlay down completely. scenarios.conversation_overlay_lifecycle = async () => { const harness = makeHarness(); await harness.start(); const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const find = (root, needle) => root.children.find( (child) => String(child.className).indexOf(needle) >= 0, ); const role = overlay.getAttribute('role'); const ariaModal = overlay.getAttribute('aria-modal'); await harness.silence(300); const listeningState = overlay.dataset.voiceState; await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); const captions = find(overlay, 'voice-conversation-captions'); const userCaption = captions ? captions.children[0].textContent : ''; const captionsLive = captions ? captions.getAttribute('aria-live') : ''; harness.setAssistantReply('A visible answer.'); await harness.tick(300); const assistantCaption = captions ? captions.children[1].textContent : ''; const thinkingState = overlay.dataset.voiceState; const controls = find(overlay, 'voice-conversation-controls'); const muteBtn = controls.children[0]; const exitBtn = controls.children[1]; muteBtn.click(); const mutedPressed = muteBtn.getAttribute('aria-pressed'); const mutedTracks = harness.trackStates(); muteBtn.click(); const unmutedTracks = harness.trackStates(); exitBtn.click(); await harness.flush(); const removedOnExit = !harness.overlay(); const inactiveAfterExit = !harness.context._voiceModeActive(); // Re-activate and leave through Escape instead. harness.elements.btnVoiceMode.click(); await harness.flush(); const overlayAgain = harness.overlay(); const keydown = overlayAgain.listeners.find((entry) => entry.type === 'keydown'); keydown.handler({ key: 'Escape', preventDefault() {} }); await harness.flush(); return { overlayPresent: true, role, ariaModal, listeningState, thinkingState, userCaption, assistantCaption, captionsLive, mutedPressed, mutedTracks, unmutedTracks, removedOnExit, inactiveAfterExit, removedOnEscape: !harness.overlay(), sends: harness.sends, }; }; // Dynamic endpointing: a >=3-word partial reads as a plausibly complete // utterance and endpoints at the base silence window; 1-2 word partials keep // the young-utterance hold so thinking pauses still never clip. scenarios.three_word_partial_endpoints_at_base_silence = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 300); await harness.speak('bravo', 300); await harness.speak('charlie', 400); await harness.silence(1200); // past the base 1100ms endpoint, well under the 1800ms hold await harness.tick(300); return { sends: harness.sends, state: harness.state() }; }; scenarios.two_word_young_utterance_keeps_the_hold = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 300); await harness.speak('bravo', 300); await harness.silence(1300); // longer than base, shorter than the hold const sendsEarly = harness.sends.slice(); await harness.silence(700); await harness.tick(300); return { sendsEarly, sends: harness.sends }; }; // An errored turn (error/system envelope in the transcript) is never spoken, // never captioned as a reply, and fully resynchronizes capture. scenarios.errored_turn_is_not_spoken_and_capture_resyncs = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); // dispatched; thinking const serversBefore = harness.servers.length; const ttsBefore = harness.ttsCalls.length; harness.setAssistantError('**Task cancelled:** Task cancelled.'); await harness.tick(300); // the response observer sees the envelope const stateAfterError = harness.state(); const labelAfterError = harness.elements.voiceModeLabel.textContent; await harness.speak('bravo', 1300); await harness.silence(1500); await harness.tick(400); return { stateAfterError, labelAfterError, ttsDuringError: harness.ttsCalls.length - ttsBefore, freshSessions: harness.servers.length - serversBefore, sends: harness.sends, }; }; // The empty new-chat screen offers a prominent "Start conversation" button when // voice is available; it enters conversation mode through the same activate() // path as the small composer toggle. scenarios.empty_state_conversation_button_enters_voice = async () => { const harness = makeHarness(); await harness.initVoice(); const btn = harness.emptyConversationButton(); const created = !!btn; const visible = btn ? btn.style.display !== 'none' : false; const ariaLabel = btn ? btn.getAttribute('aria-label') : ''; const className = btn ? btn.className : ''; const stateBefore = harness.state(); if (btn) btn.click(); await harness.flush(); return { created, visible, ariaLabel, className, stateBefore, stateAfter: harness.state(), overlayPresent: !!harness.overlay(), }; }; // A TRANSIENT provider error (a broker 5xx/502 that blipped mid-conversation) // is auto-retried in place through the app's regenerate action instead of // dropping the user's utterance: the error is never spoken, the overlay stays // in Thinking (its cues cover the reconnect gap), the regenerate button is // clicked once, and when the retried answer arrives it is spoken normally. scenarios.transient_error_auto_retries_then_speaks = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); // dispatched; thinking const ttsBefore = harness.ttsCalls.length; harness.setAssistantTransientError('**Error:** HTTP 502: error sending request for url (http://hermes-codex-broker:9003/v1/responses)'); await harness.tick(120); // observer sees the transient error -> one auto-retry const stateAfterError = harness.state(); const labelAfterError = harness.elements.voiceModeLabel.textContent; const clicksAfterError = harness.regenerateClicks(); const ttsDuringError = harness.ttsCalls.slice(ttsBefore).map((r) => r.text); // The regenerated turn recovers with a real answer. harness.setAssistantReply('Controlled Unclassified Information.'); harness.endStream(); await harness.tick(400); return { stateAfterError, labelAfterError, clicksAfterError, ttsDuringError, ttsTexts: harness.ttsCalls.map((r) => r.text), state: harness.state(), }; }; // The auto-retry is bounded: a provider that keeps returning a transient error // is retried at most MAX_TRANSIENT_RETRIES (2) times, then the turn is dropped // and capture resynchronizes so the failure surfaces instead of looping. scenarios.transient_error_retry_is_bounded = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable'); await harness.tick(150); // retry 1 (click truncates + starts a fresh stream) harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable'); await harness.tick(150); // retry 2 harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable'); await harness.tick(150); // budget exhausted -> drop + resync return { clicks: harness.regenerateClicks(), state: harness.state(), label: harness.elements.voiceModeLabel.textContent, }; }; // Voice barge-in appends a single-line cut marker naming the sentence that // was playing, so the model knows where its reply was cut off. scenarios.barge_cut_marker_records_spoken_tail = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); // dispatched; thinking harness.setAssistantReply('The first point is ready. The second point needs many more words before it becomes another speakable chunk.'); await harness.tick(300); // first sentence reaches TTS and starts playing await harness.speak('bravo', 900); // talk over the reply await harness.silence(1900); await harness.tick(400); return { sends: harness.sends, ttsTexts: harness.ttsCalls.map((request) => request.text) }; }; // The detected utterance language re-biases the following streaming STT // session (sticky per hands-free session). scenarios.sticky_language_biases_next_stt_session = async () => { const harness = makeHarness(); await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); harness.completeResponse(); await harness.tick(400); await harness.speak('bravo', 1300); await harness.silence(1500); await harness.tick(400); return { startLanguages: harness.servers.map((socket) => socket.startLanguage) }; }; // FIX 3: choosing a language in the conversation overlay FORCES both the // streaming STT hint and the reply TTS voice for the whole session, overriding // auto-detection, until Auto is chosen again. scenarios.language_override_forces_stt_and_voice = async () => { const harness = makeHarness(); await harness.start(); const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const langWrap = overlay.children.find( (child) => String(child.className).indexOf('voice-conversation-lang') >= 0, ); const menu = langWrap ? langWrap.children.find( (child) => String(child.className).indexOf('voice-conversation-lang-menu') >= 0, ) : null; const items = menu ? menu.children : []; const ru = items.find((item) => item.getAttribute('data-lang') === 'ru'); const auto = items.find((item) => item.getAttribute('data-lang') === ''); const langBtn = langWrap ? langWrap.children.find( (child) => String(child.className).indexOf('voice-conversation-lang-btn') >= 0, ) : null; if (!ru) return { overlayPresent: true, ruItemPresent: false }; // Force Russian. ru.click(); const forcedChecked = ru.getAttribute('aria-checked'); const autoChecked = auto ? auto.getAttribute('aria-checked') : null; const btnForced = langBtn ? String(langBtn.className).indexOf('is-forced') >= 0 : false; // Run a turn with an English reply: STT hint and TTS voice must both be 'ru'. await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); harness.setAssistantReply('A short English reply. It has two sentences.'); harness.completeResponse(); await harness.tick(600); const startLanguages = harness.servers.map((socket) => socket.startLanguage); const ttsLanguages = harness.ttsCalls.map((call) => call.language); // Release back to Auto. if (auto) auto.click(); const releasedChecked = auto ? auto.getAttribute('aria-checked') : null; const btnForcedAfterAuto = langBtn ? String(langBtn.className).indexOf('is-forced') >= 0 : false; return { overlayPresent: true, ruItemPresent: true, forcedChecked, autoChecked, btnForced, startLanguages, ttsLanguages, releasedChecked, btnForcedAfterAuto, }; }; // THE FOLD REGRESSION. A quick spoken acknowledgement streams as an interim // answer segment while a tool call is prepared. The pump chunks the first two // sentences and RETAINS the still-streaming third as an unspoken tail; then the // renderer folds the interim into the hidden .assistant-segment-worklog-source // (extraction → '') and the final answer renders as its own new segment. // // Before the fix: the empty-read branch returned without flushing the retained // tail (the acknowledgement's last words were dropped) and the speech queue // drained with the state fallen back to Thinking while a tail was still // outstanding; the final answer then resurfaced from the interim's stale offset // (a garbled mid-string slice). After the fix: the whole acknowledgement is // spoken, in order, BEFORE the distinct final answer, and the state never falls // back to Thinking with an unspoken interim tail outstanding. scenarios.interim_ack_fold_flushes_tail_before_final = async () => { const harness = makeHarness(); // The stub Audio never fires onended, so playback of the first chunk parks and // the queue does not drain (the same reason barge_cut_marker only inspects the // first synthesized chunk). The complete ordered set of chunks that WILL be // spoken is therefore the chunks already handed to synthesis (ttsTexts, in // order) followed by the chunks still queued behind the parked one, assembled // as spokenOrder below. Every assertion is a QUEUE/STATE fact, independent of // the playback stub. const violations = []; const record = () => { const snap = harness.speechTurnSnapshot(); if (snap && snap.state === 'thinking' && !snap.final && snap.consumed < snap.sourceLength) { violations.push({ consumed: snap.consumed, sourceLength: snap.sourceLength }); } }; const settle = async (steps) => { for (let i = 0; i < steps; i += 1) { await harness.tick(100); record(); } }; await harness.start(); await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); // dispatched; thinking; response observer running // Interim acknowledgement: two complete sentences plus a third still mid-word, // so the pump enqueues S1+S2 and retains "One moment while I che" unspoken. const interim = 'Sure thing. Let me look that up. One moment while I che'; harness.setInterimAck(interim); await settle(4); // The tool call begins: fold the interim into the hidden worklog source. harness.foldInterimAck(); await settle(6); const afterFold = harness.speechTurnSnapshot(); const queuedAfterFold = (afterFold && afterFold.queue) || []; // The final answer renders as its own new, distinct segment. const finalAnswer = 'The weather today is sunny and warm. Enjoy your afternoon out there.'; harness.setFinalAnswer(finalAnswer); await settle(6); harness.completeResponse(); await settle(10); const finalSnap = harness.speechTurnSnapshot(); const residualQueue = (finalSnap && finalSnap.queue) || []; // The full ordered list of chunks that reach the speech queue for this turn. const spokenOrder = harness.ttsTexts().concat(residualQueue); const joined = spokenOrder.join(''); const lastInterimIdx = (() => { let idx = -1; spokenOrder.forEach((t, i) => { if (/Sure thing|look that up|One moment|while I che/.test(t)) idx = i; }); return idx; })(); const firstFinalIdx = spokenOrder.findIndex((t) => /weather|sunny|afternoon/.test(t)); return { spokenOrder, // After the fold, the interim's retained tail is the next thing queued, and // it is queued BEFORE any final-answer chunk exists. queuedAfterFold, tailQueuedAfterFold: queuedAfterFold.some((t) => /while I che/.test(t)), noFinalBeforeFold: !queuedAfterFold.some((t) => /weather|sunny|afternoon/.test(t)), // (i) every interim sentence reaches the speech queue: none dropped. interimS1Reached: /Sure thing/.test(joined), interimS2Reached: /look that up/.test(joined), interimTailReached: /while I che/.test(joined), // (iii) the distinct final answer is fully queued too. finalS1Reached: /weather today is sunny/.test(joined), finalS2Reached: /Enjoy your afternoon/.test(joined), // The final answer is queued from its OWN start, never resurfaced from the // interim's stale offset ("nny and warm" is the buggy mid-string slice). finalNotGarbled: !spokenOrder.some((t) => /^nny and warm/.test(t)), // (ii) the interim finishes before the final message's chunks are enqueued. interimBeforeFinal: lastInterimIdx >= 0 && firstFinalIdx >= 0 && lastInterimIdx < firstFinalIdx, // (iv) the state never fell back to Thinking with an unspoken interim tail. thinkingWithUnspokenTail: violations.length, state: harness.state(), }; }; // UNIFIED OUTPUT SINK + device selector. When the browser can enumerate outputs // and setSinkId, a tidy corner control lists them; choosing one routes ALL spoken // output (the reply's blob element here, and — via the shared AudioContext — the // PCM reply and thinking cues) to that single device. When the APIs are missing, // the control hides entirely (a dead control never appears). scenarios.output_device_selector_routes_spoken_output = async () => { const harness = makeHarness({ outputs: [ { deviceId: '', kind: 'audiooutput', label: 'System default' }, { deviceId: 'spk-1', kind: 'audiooutput', label: 'Speaker One' }, { deviceId: 'spk-2', kind: 'audiooutput', label: 'Headphones Two' }, { deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' }, ], }); await harness.start(); await harness.flush(); await harness.flush(); const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const outWrap = overlay.children.find( (child) => String(child.className).indexOf('voice-conversation-out') >= 0, ); const outBtn = outWrap ? outWrap.children.find( (child) => String(child.className).indexOf('voice-conversation-out-btn') >= 0, ) : null; const outMenu = outWrap ? outWrap.children.find( (child) => String(child.className).indexOf('voice-conversation-out-menu') >= 0, ) : null; const shownAfterRefresh = outWrap ? outWrap.style.display !== 'none' : false; const items = outMenu ? outMenu.children : []; const itemLabels = items.map((item) => item.textContent); const headphones = items.find((item) => item.getAttribute('data-device') === 'spk-2'); if (headphones) headphones.click(); const forcedChecked = headphones ? headphones.getAttribute('aria-checked') : null; const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false; // A reply now plays through the blob fallback element, which must be routed to // the chosen sink. await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); harness.setAssistantReply('A short spoken reply.'); harness.completeResponse(); await harness.tick(400); const sinks = harness.sinkCalls(); return { overlayPresent: true, supported: true, shownAfterRefresh, itemLabels, forcedChecked, btnForced, audioRoutedTo: sinks.filter((c) => c.kind === 'audio').map((c) => c.sink), }; }; // When enumerateDevices/setSinkId are unavailable the selector is present in the // DOM but hidden — never a dead control. scenarios.output_selector_hidden_when_unsupported = async () => { const harness = makeHarness(); await harness.start(); await harness.flush(); const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const outWrap = overlay.children.find( (child) => String(child.className).indexOf('voice-conversation-out') >= 0, ); return { overlayPresent: true, controlInDom: !!outWrap, hidden: outWrap ? outWrap.style.display === 'none' : null, }; }; // Shape of the attached full-screen overlay: the essential orb + captions + // mute/exit that MUST render whenever conversation mode opens, independent of the // language/output corner selectors. function overlayShape(harness) { const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const kids = overlay.children; const find = (sub) => kids.find((c) => String(c.className).indexOf(sub) >= 0) || null; const orb = find('voice-conversation-orb'); const orbMark = orb ? orb.children.find((c) => String(c.className).indexOf('voice-conversation-orb-mark') >= 0) : null; const captions = find('voice-conversation-captions'); const controls = find('voice-conversation-controls'); const mute = controls ? controls.children.find((c) => String(c.className).indexOf('voice-conversation-mute') >= 0) : null; const exit = controls ? controls.children.find((c) => String(c.className).indexOf('voice-conversation-exit') >= 0) : null; return { overlayPresent: true, isDialog: overlay.getAttribute('role') === 'dialog', hasOrb: !!orb, hasOrbMark: !!orbMark, hasCaptions: !!captions, captionCount: captions ? captions.children.length : 0, hasMute: !!mute, hasExit: !!exit, }; } // REGRESSION LOCK: the full-screen overlay must ALWAYS attach when conversation // mode opens, even when the audio-output APIs are hostile — mediaDevices absent, // or enumerateDevices rejecting. A failure here previously discarded the whole // overlay and dropped back to the inline voice bar. scenarios.overlay_opens_despite_hostile_output_apis = async () => { const results = {}; // (A) navigator.mediaDevices is undefined when the overlay is built. { const harness = makeHarness(); await harness.flush(); const internals = harness.context.window.__atlasVoiceInternals; harness.context.navigator.mediaDevices = undefined; internals.openConversationOverlay(); results.mediaDevicesUndefined = overlayShape(harness); const overlay = harness.overlay(); const outWrap = overlay ? overlay.children.find((c) => String(c.className).indexOf('voice-conversation-out') >= 0) : null; results.mediaDevicesUndefined.outputHidden = outWrap ? outWrap.style.display === 'none' : null; internals.removeConversationOverlay(); } // (B) enumerateDevices is present (routing LOOKS supported) but REJECTS. { const harness = makeHarness(); await harness.flush(); const internals = harness.context.window.__atlasVoiceInternals; harness.context.navigator.mediaDevices.enumerateDevices = async () => { throw new Error('enumerate blocked'); }; internals.openConversationOverlay(); // The overlay must be attached synchronously, before the async rejection. const immediate = overlayShape(harness); await harness.flush(); await harness.flush(); results.enumerateRejects = overlayShape(harness); results.enumerateRejects.attachedBeforeReject = immediate.overlayPresent; internals.removeConversationOverlay(); } return results; }; // FIX 2(a): with the mic open the OS routes the system default to the earpiece; // the shared sink must auto-default to the LOUDSPEAKER (never "communications"), // with no user interaction — and that default must reach real playback. scenarios.output_defaults_to_loudspeaker = async () => { const harness = makeHarness({ outputs: [ { deviceId: '', kind: 'audiooutput', label: 'System default' }, { deviceId: 'communications', kind: 'audiooutput', label: 'Communications' }, { deviceId: 'ear-1', kind: 'audiooutput', label: 'Earpiece' }, { deviceId: 'spk-1', kind: 'audiooutput', label: 'Speakerphone' }, { deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' }, ], }); await harness.start(); await harness.flush(); await harness.flush(); const overlay = harness.overlay(); if (!overlay) return { overlayPresent: false }; const outWrap = overlay.children.find((c) => String(c.className).indexOf('voice-conversation-out') >= 0); const outBtn = outWrap ? outWrap.children.find((c) => String(c.className).indexOf('voice-conversation-out-btn') >= 0) : null; const outMenu = outWrap ? outWrap.children.find((c) => String(c.className).indexOf('voice-conversation-out-menu') >= 0) : null; const items = outMenu ? outMenu.children : []; const defaultChecked = items .filter((i) => i.getAttribute('aria-checked') === 'true') .map((i) => i.getAttribute('data-device')); const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false; const btnLabel = outBtn ? outBtn.getAttribute('aria-label') : null; // The default must reach real playback with NO user interaction: a reply now // plays through the blob fallback and must route to the auto-selected speaker. await harness.silence(300); await harness.speak('alpha', 1300); await harness.silence(1500); await harness.tick(400); harness.setAssistantReply('A short spoken reply.'); harness.completeResponse(); await harness.tick(400); const audioRoutedTo = harness.sinkCalls().filter((c) => c.kind === 'audio').map((c) => c.sink); return { overlayPresent: true, defaultChecked, btnForced, btnLabel, audioRoutedTo, }; }; // FIX 2(a) pure logic: the loudspeaker chooser prefers a labelled speaker, never // the "communications" endpoint, falls back to the first concrete non-earpiece // output, and returns '' (system default) when only an earpiece/comms exists. scenarios.loudspeaker_selection_logic = async () => { const harness = makeHarness(); await harness.flush(); const pick = harness.context.window.__atlasVoiceInternals.pickLoudspeakerSink; return { labelledSpeaker: pick([ { deviceId: 'ear', kind: 'audiooutput', label: 'Earpiece' }, { deviceId: 'spk', kind: 'audiooutput', label: 'Speakerphone' }, ]), skipsCommunications: pick([ { deviceId: 'communications', kind: 'audiooutput', label: 'Communications' }, { deviceId: 'spk', kind: 'audiooutput', label: 'Speaker' }, ]), concreteFallback: pick([ { deviceId: 'default', kind: 'audiooutput', label: '' }, { deviceId: 'dev-9', kind: 'audiooutput', label: '' }, ]), onlyEarpiece: pick([ { deviceId: 'communications', kind: 'audiooutput', label: 'Communications' }, { deviceId: 'ear', kind: 'audiooutput', label: 'Earpiece' }, ]), ignoresInputs: pick([ { deviceId: 'mic', kind: 'audioinput', label: 'Speaker Mic' }, { deviceId: 'spk', kind: 'audiooutput', label: 'Loudspeaker' }, ]), }; }; (async () => { const output = {}; for (const name of Object.keys(scenarios)) { // eslint-disable-next-line no-await-in-loop output[name] = await scenarios[name](); } process.stdout.write(JSON.stringify(output, null, 1)); })().catch((error) => { process.stderr.write(String((error && error.stack) || error)); process.exit(1); });