a071d091b0
Conversation mode was only reachable via a small icon by the composer, which is easy to miss on a fresh session. Inject a prominent, centred 'Start conversation' button into the empty new-chat state (#emptyState), below the subtitle and above the suggestions, so it sits in the vertical centre of the screen. It is created only when local voice is available, honours the same show/hide preference as the composer toggle, and enters conversation mode through the same activate() path. Fully guarded so it degrades to nothing if the empty state is absent. New probe scenario verifies it is created, visible, and opens the conversation overlay on press. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
1566 lines
60 KiB
JavaScript
1566 lines
60 KiB
JavaScript
// Deterministic continuous-capture probe for dockerfiles/hermes-webui-atlas-voice.js.
|
|
//
|
|
// Drives the real script through complete streaming hands-free turns against a
|
|
// stub STT WebSocket whose VAD, speculative EOS-freeze, resume and commit
|
|
// semantics faithfully mirror dockerfiles/hermes-jetson-stt-server.py. Words
|
|
// are encoded as distinct constant PCM amplitudes so the "transcript" of any
|
|
// audio snapshot is exactly the ordered set of words whose samples survived
|
|
// endpointing — a first-word-only regression is directly observable as a
|
|
// truncated transcript in the sent composer text.
|
|
//
|
|
// node hermes_voice_capture_probe.js <path-to-atlas-voice.js>
|
|
//
|
|
// Prints one JSON object per scenario to stdout; the python wrapper asserts.
|
|
'use strict';
|
|
|
|
const fs = require('fs');
|
|
const vm = require('vm');
|
|
|
|
const SCRIPT_PATH = process.argv[2];
|
|
if (!SCRIPT_PATH) {
|
|
throw new Error('usage: hermes_voice_capture_probe.js <atlas-voice.js>');
|
|
}
|
|
const SOURCE = fs.readFileSync(SCRIPT_PATH, 'utf8');
|
|
|
|
class FakeURL extends URL {}
|
|
FakeURL.createObjectURL = () => 'blob:capture-probe';
|
|
FakeURL.revokeObjectURL = () => {};
|
|
|
|
const SAMPLE_RATE = 16000;
|
|
const TICK_MS = 100;
|
|
const TICK_SAMPLES = (SAMPLE_RATE * TICK_MS) / 1000;
|
|
|
|
// Server constants mirrored from hermes-jetson-stt-server.py.
|
|
const SERVER_END_SILENCE_SAMPLES = (SAMPLE_RATE * 650) / 1000;
|
|
const SERVER_TAIL_SAMPLES = (SAMPLE_RATE * 220) / 1000;
|
|
|
|
function flush() {
|
|
return new Promise((resolve) => {
|
|
let hops = 0;
|
|
(function hop() {
|
|
hops += 1;
|
|
if (hops > 16) { resolve(); return; }
|
|
setImmediate(hop);
|
|
})();
|
|
});
|
|
}
|
|
|
|
function makeElement(id) {
|
|
const element = {
|
|
id,
|
|
value: '',
|
|
textContent: '',
|
|
className: '',
|
|
dataset: {},
|
|
attributes: {},
|
|
children: [],
|
|
style: {
|
|
values: new Map(),
|
|
display: '',
|
|
setProperty(name, value) { this.values.set(name, String(value)); },
|
|
getPropertyValue(name) { return this.values.get(name) || ''; },
|
|
removeProperty(name) { this.values.delete(name); },
|
|
},
|
|
listeners: [],
|
|
setAttribute(name, value) { this.attributes[name] = String(value); },
|
|
getAttribute(name) {
|
|
return Object.prototype.hasOwnProperty.call(this.attributes, name)
|
|
? this.attributes[name] : null;
|
|
},
|
|
addEventListener(type, handler) { this.listeners.push({ type, handler }); },
|
|
removeEventListener(type, handler) {
|
|
element.listeners = element.listeners.filter((entry) => entry.handler !== handler);
|
|
},
|
|
appendChild(child) { element.children.push(child); child.parentNode = element; return child; },
|
|
removeChild(child) {
|
|
element.children = element.children.filter((entry) => entry !== child);
|
|
child.parentNode = null;
|
|
return child;
|
|
},
|
|
contains() { return false; },
|
|
focus() {},
|
|
querySelector() { return null; },
|
|
querySelectorAll() { return []; },
|
|
insertBefore() {},
|
|
click() {
|
|
const event = { preventDefault() {}, stopImmediatePropagation() {} };
|
|
element.listeners
|
|
.filter((entry) => entry.type === 'click')
|
|
.forEach((entry) => entry.handler(event));
|
|
},
|
|
};
|
|
element.classList = {
|
|
add(name) {
|
|
const names = new Set(element.className.split(/\s+/).filter(Boolean));
|
|
names.add(name);
|
|
element.className = [...names].join(' ');
|
|
},
|
|
remove(name) {
|
|
element.className = element.className
|
|
.split(/\s+/).filter((value) => value && value !== name).join(' ');
|
|
},
|
|
contains(name) { return element.className.split(/\s+/).includes(name); },
|
|
toggle(name, force) {
|
|
const present = this.contains(name);
|
|
const next = force === undefined ? !present : !!force;
|
|
if (next) this.add(name); else this.remove(name);
|
|
return next;
|
|
},
|
|
};
|
|
return element;
|
|
}
|
|
|
|
// ── Faithful assistant-turn DOM ─────────────────────────────────────────────
|
|
// A minimal but real querySelectorAll / closest / getAttribute / hidden /
|
|
// recursive-textContent implementation, so the interim-acknowledgement FOLD can
|
|
// be reproduced exactly as the live renderer performs it: the interim segment is
|
|
// re-tagged .assistant-segment-worklog-source + hidden + aria-hidden, at which
|
|
// point the script's real collectAssistantResponse() returns '' for it (the same
|
|
// extraction the response probe locks) and the final answer renders as its own
|
|
// new .assistant-segment. Supports the compound selectors the extraction uses.
|
|
function parseDomSelector(selector) {
|
|
return String(selector).split(',').map((group) => {
|
|
const term = group.trim();
|
|
const parts = [];
|
|
const re = /([.#]?[\w-]+)|\[([\w-]+)(?:([~|^$*]?=)"?([^"\]]*)"?)?\]/g;
|
|
let m;
|
|
while ((m = re.exec(term))) {
|
|
if (m[1]) {
|
|
if (m[1][0] === '.') parts.push({ kind: 'class', value: m[1].slice(1) });
|
|
else parts.push({ kind: 'tag', value: m[1].toLowerCase() });
|
|
} else if (m[2]) {
|
|
parts.push({ kind: 'attr', name: m[2], op: m[3] || null, value: m[4] });
|
|
}
|
|
}
|
|
return parts;
|
|
}).filter((parts) => parts.length);
|
|
}
|
|
|
|
class DomNode {
|
|
constructor(tag) {
|
|
this.tag = String(tag || 'div').toLowerCase();
|
|
this.className = '';
|
|
this.attributes = new Map();
|
|
this.children = [];
|
|
this.parentNode = null;
|
|
this.hidden = false;
|
|
this._text = '';
|
|
const self = this;
|
|
this.dataset = new Proxy({}, {
|
|
get(_t, key) {
|
|
if (typeof key !== 'string') return undefined;
|
|
const attr = 'data-' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
|
|
return self.attributes.has(attr) ? self.attributes.get(attr) : undefined;
|
|
},
|
|
has(_t, key) {
|
|
const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
|
|
return self.attributes.has(attr);
|
|
},
|
|
});
|
|
}
|
|
get classList() {
|
|
const el = this;
|
|
return {
|
|
add(name) { const s = new Set(el.className.split(/\s+/).filter(Boolean)); s.add(name); el.className = [...s].join(' '); },
|
|
remove(name) { el.className = el.className.split(/\s+/).filter((v) => v && v !== name).join(' '); },
|
|
contains(name) { return el.className.split(/\s+/).indexOf(name) >= 0; },
|
|
};
|
|
}
|
|
setAttribute(name, value) {
|
|
this.attributes.set(name, String(value));
|
|
if (name === 'class') this.className = String(value);
|
|
if (name === 'hidden') this.hidden = true;
|
|
}
|
|
getAttribute(name) {
|
|
if (name === 'class') return this.className || null;
|
|
return this.attributes.has(name) ? this.attributes.get(name) : null;
|
|
}
|
|
appendChild(child) { child.parentNode = this; this.children.push(child); return child; }
|
|
set textContent(value) { this._text = String(value); this.children = []; }
|
|
get textContent() {
|
|
if (this.children.length) return this.children.map((c) => c.textContent).join('');
|
|
return this._text;
|
|
}
|
|
_matchesTerm(parts) {
|
|
return parts.every((p) => {
|
|
if (p.kind === 'class') return this.classList.contains(p.value);
|
|
if (p.kind === 'tag') return this.tag === p.value;
|
|
if (p.kind === 'attr') {
|
|
if (!this.attributes.has(p.name)) return false;
|
|
return p.op ? this.attributes.get(p.name) === p.value : true;
|
|
}
|
|
return false;
|
|
});
|
|
}
|
|
matches(selector) { return parseDomSelector(selector).some((parts) => this._matchesTerm(parts)); }
|
|
_walk(out) { for (const c of this.children) { out.push(c); c._walk(out); } return out; }
|
|
querySelectorAll(selector) {
|
|
const groups = parseDomSelector(selector);
|
|
return this._walk([]).filter((node) => groups.some((parts) => node._matchesTerm(parts)));
|
|
}
|
|
querySelector(selector) { const all = this.querySelectorAll(selector); return all.length ? all[0] : null; }
|
|
closest(selector) {
|
|
const groups = parseDomSelector(selector);
|
|
let node = this;
|
|
while (node) { if (groups.some((parts) => node._matchesTerm(parts))) return node; node = node.parentNode; }
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function domEl(tag, className, attrs, text) {
|
|
const node = new DomNode(tag);
|
|
if (className) node.setAttribute('class', className);
|
|
if (attrs) Object.keys(attrs).forEach((k) => node.setAttribute(k, attrs[k]));
|
|
if (text !== undefined) node.textContent = text;
|
|
return node;
|
|
}
|
|
|
|
// A rendered assistant turn: role header, worklog chip, and an assistant-turn
|
|
// blocks container the interim/final segments hang off of.
|
|
function buildAssistantTurn() {
|
|
const turn = domEl('div', 'msg-row assistant-turn', { 'data-role': 'assistant', 'data-session-id': 'session-1' });
|
|
const role = domEl('div', 'msg-role assistant');
|
|
role.appendChild(domEl('div', 'role-icon assistant', null, 'H'));
|
|
role.appendChild(domEl('span', 'msg-role-name', null, 'Hermes'));
|
|
turn.appendChild(role);
|
|
const blocks = domEl('div', 'assistant-turn-blocks');
|
|
turn.appendChild(blocks);
|
|
turn.blocks = blocks;
|
|
return turn;
|
|
}
|
|
|
|
// A live (still-streaming) answer segment: no data-raw-text yet, so the script
|
|
// reads its answer .msg-body — exactly the interim acknowledgement path.
|
|
function addLiveAnswerSegment(turn, text) {
|
|
const seg = domEl('div', 'assistant-segment', { 'data-live-assistant': '1' });
|
|
seg.appendChild(domEl('div', 'msg-body', null, text));
|
|
turn.blocks.appendChild(seg);
|
|
return seg;
|
|
}
|
|
|
|
// A settled answer segment carrying the renderer-stamped clean answer text.
|
|
function addSettledAnswerSegment(turn, text) {
|
|
const seg = domEl('div', 'assistant-segment', { 'data-raw-text': text });
|
|
seg.appendChild(domEl('div', 'msg-body', null, text));
|
|
turn.blocks.appendChild(seg);
|
|
return seg;
|
|
}
|
|
|
|
// Fold an interim segment into the hidden worklog source, exactly as the live
|
|
// renderer does when a tool call begins: the extraction now excludes it.
|
|
function foldSegmentIntoWorklog(seg) {
|
|
seg.setAttribute('class', 'assistant-segment assistant-segment-worklog-source');
|
|
seg.setAttribute('aria-hidden', 'true');
|
|
seg.hidden = true;
|
|
}
|
|
|
|
// Faithful port of StreamingTranscription's endpointing/speculation contract.
|
|
class StubSttServer {
|
|
constructor(words) {
|
|
// words: {amplitude(int16) -> word string}
|
|
this.words = words;
|
|
this.chunks = []; // Int16Array chunks in arrival order
|
|
this.totalSamples = 0;
|
|
this.noiseFloor = 0.004;
|
|
this.heardSpeech = false;
|
|
this.atEos = false;
|
|
this.silenceSamples = 0;
|
|
this.lastSpeechSample = 0;
|
|
this.clientActive = false;
|
|
this.epoch = 0;
|
|
this.frozenSnapshotSamples = -1;
|
|
this.committed = false;
|
|
this.events = [];
|
|
}
|
|
|
|
snapshotSamples() {
|
|
if (this.atEos && this.frozenSnapshotSamples >= 0) return this.frozenSnapshotSamples;
|
|
let cutoff = this.totalSamples;
|
|
if (this.heardSpeech && this.lastSpeechSample) {
|
|
cutoff = Math.min(cutoff, this.lastSpeechSample + SERVER_TAIL_SAMPLES);
|
|
}
|
|
return cutoff;
|
|
}
|
|
|
|
append(int16) {
|
|
if (this.committed) throw new Error('server: audio after commit');
|
|
this.chunks.push(int16);
|
|
this.totalSamples += int16.length;
|
|
let energy = 0;
|
|
for (let index = 0; index < int16.length; index += 1) {
|
|
const value = int16[index] / 32768;
|
|
energy += value * value;
|
|
}
|
|
const rms = int16.length ? Math.sqrt(energy / int16.length) : 0;
|
|
const threshold = Math.max(0.012, this.noiseFloor * 2.5 + 0.004);
|
|
const speech = this.clientActive || rms >= threshold;
|
|
if (!this.heardSpeech && !speech) {
|
|
this.noiseFloor = this.noiseFloor * 0.96 + rms * 0.04;
|
|
}
|
|
if (speech) {
|
|
if (this.atEos) {
|
|
this.epoch += 1;
|
|
this.frozenSnapshotSamples = -1;
|
|
this.events.push('recover-by-rms');
|
|
}
|
|
this.heardSpeech = true;
|
|
this.atEos = false;
|
|
this.silenceSamples = 0;
|
|
this.lastSpeechSample = this.totalSamples;
|
|
} else if (this.heardSpeech) {
|
|
this.silenceSamples += int16.length;
|
|
if (this.silenceSamples >= SERVER_END_SILENCE_SAMPLES && !this.atEos) {
|
|
this.atEos = true;
|
|
this.frozenSnapshotSamples = this.snapshotSamples();
|
|
this.events.push('auto-eos-freeze');
|
|
}
|
|
}
|
|
}
|
|
|
|
speculate() {
|
|
this.clientActive = false;
|
|
this.atEos = true;
|
|
this.frozenSnapshotSamples = this.snapshotSamples();
|
|
this.events.push('speculate-freeze');
|
|
}
|
|
|
|
resume() {
|
|
this.atEos = false;
|
|
this.silenceSamples = 0;
|
|
this.epoch += 1;
|
|
this.clientActive = true;
|
|
this.frozenSnapshotSamples = -1;
|
|
this.events.push('resume-clear');
|
|
}
|
|
|
|
transcript() {
|
|
const cutoff = this.snapshotSamples();
|
|
const decoded = [];
|
|
let position = 0;
|
|
let currentWord = null;
|
|
let runLength = 0;
|
|
const flushRun = () => {
|
|
// Ignore sub-40ms runs: resampler transition samples, not words.
|
|
if (currentWord && runLength >= SAMPLE_RATE * 0.04) decoded.push(currentWord);
|
|
currentWord = null;
|
|
runLength = 0;
|
|
};
|
|
for (const chunk of this.chunks) {
|
|
for (let index = 0; index < chunk.length; index += 1) {
|
|
if (position >= cutoff) { flushRun(); return decoded.join(' '); }
|
|
position += 1;
|
|
const magnitude = Math.abs(chunk[index]);
|
|
let word = null;
|
|
for (const [amplitude, name] of Object.entries(this.words)) {
|
|
if (Math.abs(magnitude - Number(amplitude)) <= 200) { word = name; break; }
|
|
}
|
|
if (word === currentWord) {
|
|
runLength += 1;
|
|
} else {
|
|
flushRun();
|
|
currentWord = word;
|
|
runLength = word ? 1 : 0;
|
|
}
|
|
}
|
|
}
|
|
flushRun();
|
|
return decoded.join(' ');
|
|
}
|
|
}
|
|
|
|
function makeHarness(options = {}) {
|
|
const clock = { now: 1000000 };
|
|
const timeouts = [];
|
|
const intervals = new Map();
|
|
let timerId = 1;
|
|
|
|
const sends = [];
|
|
const toasts = [];
|
|
const transcribeUploads = [];
|
|
const ttsCalls = [];
|
|
const servers = [];
|
|
const captureNodes = [];
|
|
const analysers = [];
|
|
const recorders = [];
|
|
let assistantRows = [];
|
|
let streamCounter = 0;
|
|
|
|
const words = options.words || { 8000: 'alpha', 12000: 'bravo', 16000: 'charlie' };
|
|
const micTracks = [];
|
|
const sinkCalls = [];
|
|
|
|
const elements = {};
|
|
['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg', 'emptyState']
|
|
.forEach((id) => { elements[id] = makeElement(id); });
|
|
const bodyElement = makeElement('body');
|
|
|
|
// ── Microphone model ────────────────────────────────────────────────────
|
|
// Per tick the mic has one (analyser level, pcm amplitude) pair. The level
|
|
// feeds the script's VAD analyser; the amplitude becomes the PCM the
|
|
// capture worklet delivers to the streaming session.
|
|
const mic = { level: 0, amplitude: 0 };
|
|
|
|
class FakeAnalyser {
|
|
constructor() { this.fftSize = 0; analysers.push(this); }
|
|
getByteTimeDomainData(samples) {
|
|
for (let index = 0; index < samples.length; index += 1) {
|
|
const offset = Math.round(mic.level * 127);
|
|
samples[index] = 128 + (index % 2 ? offset : -offset);
|
|
}
|
|
}
|
|
disconnect() {}
|
|
}
|
|
|
|
class FakeAudioContext {
|
|
constructor() {
|
|
this.sampleRate = SAMPLE_RATE;
|
|
this.destination = {};
|
|
this.audioWorklet = { addModule: async () => {} };
|
|
this.closed = false;
|
|
}
|
|
createAnalyser() { return new FakeAnalyser(); }
|
|
createBiquadFilter() {
|
|
return { type: '', frequency: { value: 0 }, Q: { value: 0 }, connect() {}, disconnect() {} };
|
|
}
|
|
createMediaStreamSource() { return { connect() {}, disconnect() {} }; }
|
|
createGain() { return { gain: { value: 0, setTargetAtTime() {} }, connect() {}, disconnect() {} }; }
|
|
resume() { return Promise.resolve(); }
|
|
setSinkId(id) { sinkCalls.push({ kind: 'context', sink: id }); return Promise.resolve(); }
|
|
close() { this.closed = true; return Promise.resolve(); }
|
|
}
|
|
|
|
class FakeAudioWorkletNode {
|
|
constructor(_context, name) {
|
|
this.name = name;
|
|
this.connected = true;
|
|
const node = this;
|
|
this.port = {
|
|
onmessage: null,
|
|
postMessage(message) {
|
|
if ((message || {}).type === 'flush') {
|
|
queueMicrotask(() => {
|
|
if (node.port.onmessage) node.port.onmessage({ data: { type: 'flushed' } });
|
|
});
|
|
}
|
|
},
|
|
};
|
|
if (name === 'atlas-pcm-capture') captureNodes.push(this);
|
|
}
|
|
connect() {}
|
|
disconnect() { this.connected = false; }
|
|
}
|
|
|
|
class FakeMediaRecorder {
|
|
static isTypeSupported() { return true; }
|
|
constructor(stream, recorderOptions) {
|
|
this.stream = stream;
|
|
this.state = 'inactive';
|
|
this.mimeType = (recorderOptions && recorderOptions.mimeType) || 'audio/webm;codecs=opus';
|
|
this.ondataavailable = null;
|
|
this.onstop = null;
|
|
recorders.push(this);
|
|
}
|
|
start() { this.state = 'recording'; }
|
|
stop() {
|
|
if (this.state === 'inactive') return;
|
|
this.state = 'inactive';
|
|
if (this.ondataavailable) {
|
|
this.ondataavailable({ data: { size: 512, type: 'audio/webm;codecs=opus' } });
|
|
}
|
|
const recorder = this;
|
|
// Real recorders finalize asynchronously.
|
|
queueMicrotask(() => { if (recorder.onstop) recorder.onstop(); });
|
|
}
|
|
}
|
|
|
|
class FakeWebSocket {
|
|
constructor(url, protocols) {
|
|
this.url = url;
|
|
this.protocols = protocols;
|
|
this.readyState = 0;
|
|
this.bufferedAmount = 0;
|
|
this.binaryType = '';
|
|
this.onopen = null;
|
|
this.onmessage = null;
|
|
this.onerror = null;
|
|
this.onclose = null;
|
|
this.server = new StubSttServer(words);
|
|
this.turnId = '';
|
|
servers.push(this);
|
|
const socket = this;
|
|
queueMicrotask(() => {
|
|
if (socket.readyState !== 0) return;
|
|
socket.readyState = 1;
|
|
if (socket.onopen) socket.onopen();
|
|
});
|
|
}
|
|
deliver(payload) {
|
|
const socket = this;
|
|
queueMicrotask(() => {
|
|
if (socket.readyState !== 1) return;
|
|
if (socket.onmessage) socket.onmessage({ data: JSON.stringify(payload) });
|
|
});
|
|
}
|
|
send(data) {
|
|
if (this.readyState !== 1) throw new Error('socket not open');
|
|
if (typeof data === 'string') {
|
|
const message = JSON.parse(data);
|
|
if (message.type === 'start') { this.turnId = message.turn_id; this.startLanguage = message.language; return; }
|
|
if (message.type === 'speculate') { this.server.speculate(); return; }
|
|
if (message.type === 'resume') { this.server.resume(); return; }
|
|
if (message.type === 'cancel') { this.server.events.push('cancel'); return; }
|
|
if (message.type === 'commit') {
|
|
this.server.committed = true;
|
|
this.deliver({
|
|
type: 'final',
|
|
turn_id: this.turnId,
|
|
transcript: this.server.transcript(),
|
|
language: 'en',
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
this.server.append(new Int16Array(data));
|
|
// Mirror the server's rolling-partial stream: one stable partial per
|
|
// decoded-transcript change, so the client's dynamic endpoint sees the
|
|
// same signal the Jetson emits.
|
|
const partialText = this.server.transcript();
|
|
if (partialText && partialText !== this.lastPartialText) {
|
|
this.lastPartialText = partialText;
|
|
this.partialRevision = (this.partialRevision || 0) + 1;
|
|
this.deliver({
|
|
type: 'partial',
|
|
rolling: true,
|
|
turn_id: this.turnId,
|
|
revision: this.partialRevision,
|
|
transcript: partialText,
|
|
stable_transcript: partialText,
|
|
});
|
|
}
|
|
}
|
|
close(code) { this.readyState = 3; this.closeCode = code; }
|
|
}
|
|
|
|
async function fetchStub(url, init) {
|
|
if (url === '/api/transcribe/capability') {
|
|
return { ok: true, status: 200, json: async () => ({ available: true, provider: 'local_command' }) };
|
|
}
|
|
if (url === '/api/voice/streaming/capability') {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
json: async () => ({
|
|
tts: { available: false },
|
|
stt: { available: true, transport: 'websocket', format: 'pcm_s16le', sample_rate: 16000 },
|
|
preflight: { available: false },
|
|
}),
|
|
};
|
|
}
|
|
if (url === '/api/transcribe') {
|
|
transcribeUploads.push({ body: init && init.body });
|
|
return { ok: true, status: 200, json: async () => ({ transcript: 'CONTAINER-FALLBACK', language: 'en' }) };
|
|
}
|
|
if (url === '/api/tts') {
|
|
ttsCalls.push(JSON.parse((init && init.body) || '{}'));
|
|
return { ok: true, status: 200, blob: async () => ({ synthetic: true }), json: async () => ({}) };
|
|
}
|
|
if (String(url).indexOf('api/chat/cancel') >= 0) {
|
|
context.S.busy = false;
|
|
context.S.activeStreamId = null;
|
|
if (context.S.session) context.S.session.active_stream_id = null;
|
|
return { ok: true, status: 200, json: async () => ({ cancelled: true }) };
|
|
}
|
|
throw new Error(`unexpected fetch: ${url}`);
|
|
}
|
|
|
|
const storage = new Map();
|
|
const context = {
|
|
AbortController,
|
|
console,
|
|
Uint8Array,
|
|
Int16Array,
|
|
Float32Array,
|
|
ArrayBuffer,
|
|
DataView,
|
|
Promise,
|
|
Math,
|
|
JSON,
|
|
String,
|
|
Number,
|
|
Error,
|
|
parseInt,
|
|
parseFloat,
|
|
isNaN,
|
|
Set,
|
|
Map,
|
|
Array,
|
|
Object,
|
|
URL: FakeURL,
|
|
queueMicrotask,
|
|
Date: { now: () => clock.now },
|
|
Blob: function Blob(parts, blobOptions) {
|
|
this.parts = parts;
|
|
this.type = (blobOptions || {}).type || '';
|
|
this.size = 512;
|
|
},
|
|
File: function File(parts, name, fileOptions) {
|
|
this.parts = parts; this.name = name; this.type = (fileOptions || {}).type || '';
|
|
},
|
|
FormData: function FormData() {
|
|
this.entries = []; this.append = (key, value) => this.entries.push([key, value]);
|
|
},
|
|
Audio: function Audio() {
|
|
this.play = () => Promise.resolve();
|
|
this.pause = () => {};
|
|
this.setSinkId = (id) => { sinkCalls.push({ kind: 'audio', sink: id }); return Promise.resolve(); };
|
|
this.onended = null;
|
|
this.onerror = null;
|
|
},
|
|
WebSocket: FakeWebSocket,
|
|
AudioWorkletNode: FakeAudioWorkletNode,
|
|
MediaRecorder: FakeMediaRecorder,
|
|
AudioContext: FakeAudioContext,
|
|
fetch: fetchStub,
|
|
localStorage: {
|
|
getItem: (key) => (storage.has(key) ? storage.get(key) : null),
|
|
setItem: (key, value) => { storage.set(key, String(value)); },
|
|
removeItem: (key) => { storage.delete(key); },
|
|
},
|
|
navigator: {
|
|
mediaDevices: {
|
|
getSupportedConstraints: () => ({}),
|
|
...(options.outputs ? { enumerateDevices: async () => options.outputs } : {}),
|
|
getUserMedia: async () => {
|
|
const track = {
|
|
stop() {},
|
|
enabled: true,
|
|
getSettings: () => ({ echoCancellation: true }),
|
|
};
|
|
micTracks.push(track);
|
|
return { getTracks: () => [track], getAudioTracks: () => [track] };
|
|
},
|
|
},
|
|
},
|
|
document: {
|
|
baseURI: 'https://chat.test/',
|
|
body: bodyElement,
|
|
getElementById: (id) => elements[id] || null,
|
|
querySelectorAll: () => assistantRows,
|
|
createElement: (tag) => makeElement(`created-${tag}`),
|
|
addEventListener() {},
|
|
removeEventListener() {},
|
|
},
|
|
location: { protocol: 'https:', host: 'chat.test', href: 'https://chat.test/' },
|
|
crypto: { getRandomValues(bytes) { for (let i = 0; i < bytes.length; i += 1) bytes[i] = i + 1; } },
|
|
S: { session: { session_id: 'session-1' }, busy: false, activeStreamId: null },
|
|
setTimeout: (fn, delay) => {
|
|
const id = timerId; timerId += 1;
|
|
timeouts.push({ id, fn, at: clock.now + (delay || 0) });
|
|
return id;
|
|
},
|
|
clearTimeout: (id) => {
|
|
const index = timeouts.findIndex((entry) => entry.id === id);
|
|
if (index >= 0) timeouts.splice(index, 1);
|
|
},
|
|
setInterval: (fn, delay) => {
|
|
const id = timerId; timerId += 1;
|
|
intervals.set(id, { fn, delay: delay || 0 });
|
|
return id;
|
|
},
|
|
clearInterval: (id) => { intervals.delete(id); },
|
|
};
|
|
context.window = context;
|
|
context.window.location = context.location;
|
|
context.window.crypto = context.crypto;
|
|
context.showToast = (message) => { toasts.push(message); };
|
|
context.send = () => {
|
|
streamCounter += 1;
|
|
sends.push(elements.msg.value);
|
|
context.S.busy = true;
|
|
context.S.activeStreamId = `stream-${streamCounter}`;
|
|
if (context.S.session) context.S.session.active_stream_id = context.S.activeStreamId;
|
|
};
|
|
context.autoResize = () => {};
|
|
context.stopTTS = () => {};
|
|
context._stripForTTS = (text) => text;
|
|
|
|
vm.createContext(context);
|
|
vm.runInContext(SOURCE, context, { filename: 'atlas-voice.js' });
|
|
|
|
function runDueTimeouts() {
|
|
const due = timeouts.filter((entry) => entry.at <= clock.now);
|
|
due.forEach((entry) => {
|
|
const index = timeouts.indexOf(entry);
|
|
if (index >= 0) timeouts.splice(index, 1);
|
|
entry.fn();
|
|
});
|
|
}
|
|
|
|
function currentCaptureNode() {
|
|
for (let index = captureNodes.length - 1; index >= 0; index -= 1) {
|
|
if (captureNodes[index].connected) return captureNodes[index];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function deliverMicFrame() {
|
|
const node = currentCaptureNode();
|
|
if (!node || !node.port.onmessage) return;
|
|
const samples = new Float32Array(TICK_SAMPLES);
|
|
const value = mic.amplitude / 32768;
|
|
for (let index = 0; index < samples.length; index += 1) {
|
|
samples[index] = index % 2 ? value : -value;
|
|
}
|
|
node.port.onmessage({ data: { type: 'pcm', samples: samples.buffer } });
|
|
}
|
|
|
|
async function tick(ms) {
|
|
const steps = Math.max(1, Math.round(ms / TICK_MS));
|
|
for (let step = 0; step < steps; step += 1) {
|
|
clock.now += TICK_MS;
|
|
deliverMicFrame();
|
|
Array.from(intervals.values()).forEach((entry) => entry.fn());
|
|
runDueTimeouts();
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await flush();
|
|
}
|
|
}
|
|
|
|
async function speak(word, ms) {
|
|
const amplitude = Number(Object.keys(words).find((key) => words[key] === word));
|
|
if (!amplitude) throw new Error(`unknown word: ${word}`);
|
|
mic.level = Math.min(0.9, amplitude / 32768 + 0.25);
|
|
mic.amplitude = amplitude;
|
|
await tick(ms);
|
|
}
|
|
|
|
async function silence(ms) {
|
|
mic.level = 0;
|
|
mic.amplitude = 0;
|
|
await tick(ms);
|
|
}
|
|
|
|
return {
|
|
context,
|
|
elements,
|
|
sends,
|
|
toasts,
|
|
transcribeUploads,
|
|
ttsCalls,
|
|
servers,
|
|
recorders,
|
|
clock,
|
|
tick,
|
|
speak,
|
|
silence,
|
|
flush,
|
|
runDueTimeouts,
|
|
setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; },
|
|
setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; },
|
|
// A TRANSIENT provider error turn: an error envelope carrying a
|
|
// .provider-error-details block (so the extraction reports error), plus the
|
|
// last-assistant "regenerate" action button the auto-retry clicks. The
|
|
// button records each click so the scenario can assert the bounded retry.
|
|
setAssistantTransientError(text) {
|
|
// An errored turn is a COMPLETED turn: its stream ended, so the session is
|
|
// no longer busy (the same precondition the app's regenerate action needs).
|
|
context.S.busy = false;
|
|
context.S.activeStreamId = null;
|
|
if (context.S.session) context.S.session.active_stream_id = null;
|
|
const turn = buildAssistantTurn();
|
|
const seg = domEl('div', 'assistant-segment');
|
|
seg.appendChild(domEl('div', 'msg-body', null, text));
|
|
seg.appendChild(domEl('details', 'provider-error-details', null, text));
|
|
turn.blocks.appendChild(seg);
|
|
const foot = domEl('div', 'msg-foot');
|
|
const actions = domEl('span', 'msg-actions');
|
|
const regen = domEl('button', 'msg-action-btn', { onclick: 'regenerateResponse(this)', title: 'regenerate' });
|
|
regen.click = () => {
|
|
// Mirror the app's regenerate: truncate the errored turn and start a
|
|
// fresh stream. The scenario injects the next state (another error, or a
|
|
// recovered answer) to model whether the retried turn succeeds.
|
|
this._regenerateClicks = (this._regenerateClicks || 0) + 1;
|
|
assistantRows = [];
|
|
context.S.busy = true;
|
|
streamCounter += 1;
|
|
context.S.activeStreamId = `stream-${streamCounter}`;
|
|
};
|
|
actions.appendChild(regen);
|
|
foot.appendChild(actions);
|
|
turn.appendChild(foot);
|
|
assistantRows = [turn];
|
|
},
|
|
regenerateClicks() { return this._regenerateClicks || 0; },
|
|
endStream() { context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; },
|
|
clearAssistantRows() { assistantRows = []; },
|
|
// ── Interim-fold reproduction ─────────────────────────────────────────
|
|
// Stream an interim acknowledgement as a live answer segment, then FOLD it
|
|
// into the hidden worklog source (as the renderer does at the tool call) and
|
|
// render the final answer as its own new segment.
|
|
setInterimAck(text) {
|
|
const turn = buildAssistantTurn();
|
|
this._foldTurn = turn;
|
|
this._interimSeg = addLiveAnswerSegment(turn, text);
|
|
assistantRows = [turn];
|
|
},
|
|
foldInterimAck() { if (this._interimSeg) foldSegmentIntoWorklog(this._interimSeg); },
|
|
setFinalAnswer(text) {
|
|
if (!this._foldTurn) return;
|
|
addSettledAnswerSegment(this._foldTurn, text);
|
|
},
|
|
speechTurnSnapshot() {
|
|
const internals = context.window.__atlasVoiceInternals;
|
|
return internals && typeof internals.speechTurnSnapshot === 'function'
|
|
? internals.speechTurnSnapshot() : null;
|
|
},
|
|
ttsTexts() { return ttsCalls.map((request) => request.text); },
|
|
sinkCalls() { return sinkCalls.slice(); },
|
|
state() { return elements.voiceModeBar.dataset.voiceState || ''; },
|
|
body() { return bodyElement; },
|
|
trackStates() { return micTracks.map((track) => track.enabled); },
|
|
overlay() {
|
|
return bodyElement.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation') >= 0,
|
|
) || null;
|
|
},
|
|
completeResponse() {
|
|
context.S.busy = false;
|
|
context.S.activeStreamId = null;
|
|
if (context.S.session) context.S.session.active_stream_id = null;
|
|
context.autoReadLastAssistant();
|
|
},
|
|
async start() {
|
|
await flush();
|
|
elements.btnVoiceMode.click();
|
|
await flush();
|
|
await flush();
|
|
},
|
|
// Boot the script (runs initialize(), which injects the empty-state
|
|
// conversation button when voice is available) WITHOUT entering voice mode.
|
|
async initVoice() {
|
|
await flush();
|
|
await flush();
|
|
},
|
|
emptyConversationButton() {
|
|
const empty = elements.emptyState;
|
|
return (empty.children || []).find(
|
|
(child) => child && child.id === 'btnEmptyConversation',
|
|
) || null;
|
|
},
|
|
};
|
|
}
|
|
|
|
const scenarios = {};
|
|
|
|
// The reported live regression: a normal multi-word utterance with a natural
|
|
// inter-word pause long enough for the client to speculate (>=605ms at the
|
|
// default 1100ms endpoint) must still transcribe completely.
|
|
scenarios.multiword_utterance_with_interword_pause = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 400);
|
|
await harness.silence(700); // client speculates at ~605ms
|
|
await harness.speak('bravo', 600);
|
|
await harness.silence(1400); // past the 1100ms endpoint
|
|
await harness.tick(400); // commit retries + final delivery
|
|
return {
|
|
sends: harness.sends,
|
|
state: harness.state(),
|
|
serverEvents: harness.servers.map((socket) => socket.server.events),
|
|
toasts: harness.toasts,
|
|
};
|
|
};
|
|
|
|
// Continuous capture: the mic stays hot after an utterance is dispatched, and
|
|
// the next utterance (spoken while the first response is still in flight)
|
|
// barges in and is transcribed completely, including words after a pause.
|
|
scenarios.second_utterance_during_response_is_complete = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 500);
|
|
await harness.silence(1400);
|
|
await harness.tick(400); // first send dispatched
|
|
const firstSends = harness.sends.slice();
|
|
// The model is now answering (S.busy). Assistant text starts streaming.
|
|
harness.setAssistantReply('Partial answer already visible.');
|
|
await harness.tick(200);
|
|
// User talks over the response with a two-word interruption + pause.
|
|
await harness.speak('bravo', 500);
|
|
await harness.silence(700);
|
|
await harness.speak('charlie', 500);
|
|
await harness.silence(1400);
|
|
await harness.tick(400);
|
|
return {
|
|
firstSends,
|
|
sends: harness.sends,
|
|
state: harness.state(),
|
|
serverEvents: harness.servers.map((socket) => socket.server.events),
|
|
toasts: harness.toasts,
|
|
};
|
|
};
|
|
|
|
// The mic never re-acquires getUserMedia across utterances and a second
|
|
// utterance right after a normally completed response is sent alone: a
|
|
// normal completion clears any stitch context.
|
|
scenarios.back_to_back_utterances_stay_hot = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400); // first send dispatched (past the endpoint)
|
|
const sendsAfterFirst = harness.sends.slice();
|
|
harness.completeResponse();
|
|
await harness.tick(400);
|
|
await harness.speak('bravo', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
return {
|
|
sendsAfterFirst,
|
|
sends: harness.sends,
|
|
micAcquisitions: harness.trackStates().length,
|
|
recorderCount: harness.recorders.length,
|
|
serverCount: harness.servers.length,
|
|
toasts: harness.toasts,
|
|
};
|
|
};
|
|
|
|
// The exact live regression: "The … <thinking pause> rest of the sentence"
|
|
// used to endpoint on the first inter-word gap (>=1100ms) and send the first
|
|
// word alone. A young utterance now holds its endpoint to 1800ms, so the
|
|
// pause stays inside one utterance and one complete message is sent.
|
|
scenarios.thinking_pause_after_first_word_does_not_split = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 400);
|
|
await harness.silence(1300); // longer than the old 1100ms endpoint
|
|
await harness.speak('bravo', 700);
|
|
await harness.silence(2000);
|
|
await harness.tick(400);
|
|
return {
|
|
sends: harness.sends,
|
|
serverEvents: harness.servers.map((socket) => socket.server.events),
|
|
toasts: harness.toasts,
|
|
};
|
|
};
|
|
|
|
// Conversation mode: activation builds the full-screen overlay, captions
|
|
// follow the turn, mute drives the real capture track, and both the exit
|
|
// control and Escape tear the overlay down completely.
|
|
scenarios.conversation_overlay_lifecycle = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const find = (root, needle) => root.children.find(
|
|
(child) => String(child.className).indexOf(needle) >= 0,
|
|
);
|
|
const role = overlay.getAttribute('role');
|
|
const ariaModal = overlay.getAttribute('aria-modal');
|
|
await harness.silence(300);
|
|
const listeningState = overlay.dataset.voiceState;
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
const captions = find(overlay, 'voice-conversation-captions');
|
|
const userCaption = captions ? captions.children[0].textContent : '';
|
|
const captionsLive = captions ? captions.getAttribute('aria-live') : '';
|
|
harness.setAssistantReply('A visible answer.');
|
|
await harness.tick(300);
|
|
const assistantCaption = captions ? captions.children[1].textContent : '';
|
|
const thinkingState = overlay.dataset.voiceState;
|
|
const controls = find(overlay, 'voice-conversation-controls');
|
|
const muteBtn = controls.children[0];
|
|
const exitBtn = controls.children[1];
|
|
muteBtn.click();
|
|
const mutedPressed = muteBtn.getAttribute('aria-pressed');
|
|
const mutedTracks = harness.trackStates();
|
|
muteBtn.click();
|
|
const unmutedTracks = harness.trackStates();
|
|
exitBtn.click();
|
|
await harness.flush();
|
|
const removedOnExit = !harness.overlay();
|
|
const inactiveAfterExit = !harness.context._voiceModeActive();
|
|
// Re-activate and leave through Escape instead.
|
|
harness.elements.btnVoiceMode.click();
|
|
await harness.flush();
|
|
const overlayAgain = harness.overlay();
|
|
const keydown = overlayAgain.listeners.find((entry) => entry.type === 'keydown');
|
|
keydown.handler({ key: 'Escape', preventDefault() {} });
|
|
await harness.flush();
|
|
return {
|
|
overlayPresent: true,
|
|
role,
|
|
ariaModal,
|
|
listeningState,
|
|
thinkingState,
|
|
userCaption,
|
|
assistantCaption,
|
|
captionsLive,
|
|
mutedPressed,
|
|
mutedTracks,
|
|
unmutedTracks,
|
|
removedOnExit,
|
|
inactiveAfterExit,
|
|
removedOnEscape: !harness.overlay(),
|
|
sends: harness.sends,
|
|
};
|
|
};
|
|
|
|
// Dynamic endpointing: a >=3-word partial reads as a plausibly complete
|
|
// utterance and endpoints at the base silence window; 1-2 word partials keep
|
|
// the young-utterance hold so thinking pauses still never clip.
|
|
scenarios.three_word_partial_endpoints_at_base_silence = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 300);
|
|
await harness.speak('bravo', 300);
|
|
await harness.speak('charlie', 400);
|
|
await harness.silence(1200); // past the base 1100ms endpoint, well under the 1800ms hold
|
|
await harness.tick(300);
|
|
return { sends: harness.sends, state: harness.state() };
|
|
};
|
|
|
|
scenarios.two_word_young_utterance_keeps_the_hold = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 300);
|
|
await harness.speak('bravo', 300);
|
|
await harness.silence(1300); // longer than base, shorter than the hold
|
|
const sendsEarly = harness.sends.slice();
|
|
await harness.silence(700);
|
|
await harness.tick(300);
|
|
return { sendsEarly, sends: harness.sends };
|
|
};
|
|
|
|
// An errored turn (error/system envelope in the transcript) is never spoken,
|
|
// never captioned as a reply, and fully resynchronizes capture.
|
|
scenarios.errored_turn_is_not_spoken_and_capture_resyncs = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400); // dispatched; thinking
|
|
const serversBefore = harness.servers.length;
|
|
const ttsBefore = harness.ttsCalls.length;
|
|
harness.setAssistantError('**Task cancelled:** Task cancelled.');
|
|
await harness.tick(300); // the response observer sees the envelope
|
|
const stateAfterError = harness.state();
|
|
const labelAfterError = harness.elements.voiceModeLabel.textContent;
|
|
await harness.speak('bravo', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
return {
|
|
stateAfterError,
|
|
labelAfterError,
|
|
ttsDuringError: harness.ttsCalls.length - ttsBefore,
|
|
freshSessions: harness.servers.length - serversBefore,
|
|
sends: harness.sends,
|
|
};
|
|
};
|
|
|
|
// The empty new-chat screen offers a prominent "Start conversation" button when
|
|
// voice is available; it enters conversation mode through the same activate()
|
|
// path as the small composer toggle.
|
|
scenarios.empty_state_conversation_button_enters_voice = async () => {
|
|
const harness = makeHarness();
|
|
await harness.initVoice();
|
|
const btn = harness.emptyConversationButton();
|
|
const created = !!btn;
|
|
const visible = btn ? btn.style.display !== 'none' : false;
|
|
const ariaLabel = btn ? btn.getAttribute('aria-label') : '';
|
|
const className = btn ? btn.className : '';
|
|
const stateBefore = harness.state();
|
|
if (btn) btn.click();
|
|
await harness.flush();
|
|
return {
|
|
created,
|
|
visible,
|
|
ariaLabel,
|
|
className,
|
|
stateBefore,
|
|
stateAfter: harness.state(),
|
|
overlayPresent: !!harness.overlay(),
|
|
};
|
|
};
|
|
|
|
// A TRANSIENT provider error (a broker 5xx/502 that blipped mid-conversation)
|
|
// is auto-retried in place through the app's regenerate action instead of
|
|
// dropping the user's utterance: the error is never spoken, the overlay stays
|
|
// in Thinking (its cues cover the reconnect gap), the regenerate button is
|
|
// clicked once, and when the retried answer arrives it is spoken normally.
|
|
scenarios.transient_error_auto_retries_then_speaks = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400); // dispatched; thinking
|
|
const ttsBefore = harness.ttsCalls.length;
|
|
harness.setAssistantTransientError('**Error:** HTTP 502: error sending request for url (http://hermes-codex-broker:9003/v1/responses)');
|
|
await harness.tick(120); // observer sees the transient error -> one auto-retry
|
|
const stateAfterError = harness.state();
|
|
const labelAfterError = harness.elements.voiceModeLabel.textContent;
|
|
const clicksAfterError = harness.regenerateClicks();
|
|
const ttsDuringError = harness.ttsCalls.slice(ttsBefore).map((r) => r.text);
|
|
// The regenerated turn recovers with a real answer.
|
|
harness.setAssistantReply('Controlled Unclassified Information.');
|
|
harness.endStream();
|
|
await harness.tick(400);
|
|
return {
|
|
stateAfterError,
|
|
labelAfterError,
|
|
clicksAfterError,
|
|
ttsDuringError,
|
|
ttsTexts: harness.ttsCalls.map((r) => r.text),
|
|
state: harness.state(),
|
|
};
|
|
};
|
|
|
|
// The auto-retry is bounded: a provider that keeps returning a transient error
|
|
// is retried at most MAX_TRANSIENT_RETRIES (2) times, then the turn is dropped
|
|
// and capture resynchronizes so the failure surfaces instead of looping.
|
|
scenarios.transient_error_retry_is_bounded = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
|
|
await harness.tick(150); // retry 1 (click truncates + starts a fresh stream)
|
|
harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
|
|
await harness.tick(150); // retry 2
|
|
harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
|
|
await harness.tick(150); // budget exhausted -> drop + resync
|
|
return {
|
|
clicks: harness.regenerateClicks(),
|
|
state: harness.state(),
|
|
label: harness.elements.voiceModeLabel.textContent,
|
|
};
|
|
};
|
|
|
|
// Voice barge-in appends a single-line cut marker naming the sentence that
|
|
// was playing, so the model knows where its reply was cut off.
|
|
scenarios.barge_cut_marker_records_spoken_tail = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400); // dispatched; thinking
|
|
harness.setAssistantReply('The first point is ready. The second point needs many more words before it becomes another speakable chunk.');
|
|
await harness.tick(300); // first sentence reaches TTS and starts playing
|
|
await harness.speak('bravo', 900); // talk over the reply
|
|
await harness.silence(1900);
|
|
await harness.tick(400);
|
|
return { sends: harness.sends, ttsTexts: harness.ttsCalls.map((request) => request.text) };
|
|
};
|
|
|
|
// The detected utterance language re-biases the following streaming STT
|
|
// session (sticky per hands-free session).
|
|
scenarios.sticky_language_biases_next_stt_session = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
harness.completeResponse();
|
|
await harness.tick(400);
|
|
await harness.speak('bravo', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
return { startLanguages: harness.servers.map((socket) => socket.startLanguage) };
|
|
};
|
|
|
|
// FIX 3: choosing a language in the conversation overlay FORCES both the
|
|
// streaming STT hint and the reply TTS voice for the whole session, overriding
|
|
// auto-detection, until Auto is chosen again.
|
|
scenarios.language_override_forces_stt_and_voice = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const langWrap = overlay.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-lang') >= 0,
|
|
);
|
|
const menu = langWrap ? langWrap.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-lang-menu') >= 0,
|
|
) : null;
|
|
const items = menu ? menu.children : [];
|
|
const ru = items.find((item) => item.getAttribute('data-lang') === 'ru');
|
|
const auto = items.find((item) => item.getAttribute('data-lang') === '');
|
|
const langBtn = langWrap ? langWrap.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-lang-btn') >= 0,
|
|
) : null;
|
|
if (!ru) return { overlayPresent: true, ruItemPresent: false };
|
|
// Force Russian.
|
|
ru.click();
|
|
const forcedChecked = ru.getAttribute('aria-checked');
|
|
const autoChecked = auto ? auto.getAttribute('aria-checked') : null;
|
|
const btnForced = langBtn ? String(langBtn.className).indexOf('is-forced') >= 0 : false;
|
|
// Run a turn with an English reply: STT hint and TTS voice must both be 'ru'.
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
harness.setAssistantReply('A short English reply. It has two sentences.');
|
|
harness.completeResponse();
|
|
await harness.tick(600);
|
|
const startLanguages = harness.servers.map((socket) => socket.startLanguage);
|
|
const ttsLanguages = harness.ttsCalls.map((call) => call.language);
|
|
// Release back to Auto.
|
|
if (auto) auto.click();
|
|
const releasedChecked = auto ? auto.getAttribute('aria-checked') : null;
|
|
const btnForcedAfterAuto = langBtn ? String(langBtn.className).indexOf('is-forced') >= 0 : false;
|
|
return {
|
|
overlayPresent: true,
|
|
ruItemPresent: true,
|
|
forcedChecked,
|
|
autoChecked,
|
|
btnForced,
|
|
startLanguages,
|
|
ttsLanguages,
|
|
releasedChecked,
|
|
btnForcedAfterAuto,
|
|
};
|
|
};
|
|
|
|
// THE FOLD REGRESSION. A quick spoken acknowledgement streams as an interim
|
|
// answer segment while a tool call is prepared. The pump chunks the first two
|
|
// sentences and RETAINS the still-streaming third as an unspoken tail; then the
|
|
// renderer folds the interim into the hidden .assistant-segment-worklog-source
|
|
// (extraction → '') and the final answer renders as its own new segment.
|
|
//
|
|
// Before the fix: the empty-read branch returned without flushing the retained
|
|
// tail (the acknowledgement's last words were dropped) and the speech queue
|
|
// drained with the state fallen back to Thinking while a tail was still
|
|
// outstanding; the final answer then resurfaced from the interim's stale offset
|
|
// (a garbled mid-string slice). After the fix: the whole acknowledgement is
|
|
// spoken, in order, BEFORE the distinct final answer, and the state never falls
|
|
// back to Thinking with an unspoken interim tail outstanding.
|
|
scenarios.interim_ack_fold_flushes_tail_before_final = async () => {
|
|
const harness = makeHarness();
|
|
// The stub Audio never fires onended, so playback of the first chunk parks and
|
|
// the queue does not drain (the same reason barge_cut_marker only inspects the
|
|
// first synthesized chunk). The complete ordered set of chunks that WILL be
|
|
// spoken is therefore the chunks already handed to synthesis (ttsTexts, in
|
|
// order) followed by the chunks still queued behind the parked one, assembled
|
|
// as spokenOrder below. Every assertion is a QUEUE/STATE fact, independent of
|
|
// the playback stub.
|
|
const violations = [];
|
|
const record = () => {
|
|
const snap = harness.speechTurnSnapshot();
|
|
if (snap && snap.state === 'thinking' && !snap.final && snap.consumed < snap.sourceLength) {
|
|
violations.push({ consumed: snap.consumed, sourceLength: snap.sourceLength });
|
|
}
|
|
};
|
|
const settle = async (steps) => { for (let i = 0; i < steps; i += 1) { await harness.tick(100); record(); } };
|
|
|
|
await harness.start();
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400); // dispatched; thinking; response observer running
|
|
|
|
// Interim acknowledgement: two complete sentences plus a third still mid-word,
|
|
// so the pump enqueues S1+S2 and retains "One moment while I che" unspoken.
|
|
const interim = 'Sure thing. Let me look that up. One moment while I che';
|
|
harness.setInterimAck(interim);
|
|
await settle(4);
|
|
|
|
// The tool call begins: fold the interim into the hidden worklog source.
|
|
harness.foldInterimAck();
|
|
await settle(6);
|
|
const afterFold = harness.speechTurnSnapshot();
|
|
const queuedAfterFold = (afterFold && afterFold.queue) || [];
|
|
|
|
// The final answer renders as its own new, distinct segment.
|
|
const finalAnswer = 'The weather today is sunny and warm. Enjoy your afternoon out there.';
|
|
harness.setFinalAnswer(finalAnswer);
|
|
await settle(6);
|
|
harness.completeResponse();
|
|
await settle(10);
|
|
|
|
const finalSnap = harness.speechTurnSnapshot();
|
|
const residualQueue = (finalSnap && finalSnap.queue) || [];
|
|
// The full ordered list of chunks that reach the speech queue for this turn.
|
|
const spokenOrder = harness.ttsTexts().concat(residualQueue);
|
|
const joined = spokenOrder.join('');
|
|
const lastInterimIdx = (() => {
|
|
let idx = -1;
|
|
spokenOrder.forEach((t, i) => { if (/Sure thing|look that up|One moment|while I che/.test(t)) idx = i; });
|
|
return idx;
|
|
})();
|
|
const firstFinalIdx = spokenOrder.findIndex((t) => /weather|sunny|afternoon/.test(t));
|
|
|
|
return {
|
|
spokenOrder,
|
|
// After the fold, the interim's retained tail is the next thing queued, and
|
|
// it is queued BEFORE any final-answer chunk exists.
|
|
queuedAfterFold,
|
|
tailQueuedAfterFold: queuedAfterFold.some((t) => /while I che/.test(t)),
|
|
noFinalBeforeFold: !queuedAfterFold.some((t) => /weather|sunny|afternoon/.test(t)),
|
|
// (i) every interim sentence reaches the speech queue: none dropped.
|
|
interimS1Reached: /Sure thing/.test(joined),
|
|
interimS2Reached: /look that up/.test(joined),
|
|
interimTailReached: /while I che/.test(joined),
|
|
// (iii) the distinct final answer is fully queued too.
|
|
finalS1Reached: /weather today is sunny/.test(joined),
|
|
finalS2Reached: /Enjoy your afternoon/.test(joined),
|
|
// The final answer is queued from its OWN start, never resurfaced from the
|
|
// interim's stale offset ("nny and warm" is the buggy mid-string slice).
|
|
finalNotGarbled: !spokenOrder.some((t) => /^nny and warm/.test(t)),
|
|
// (ii) the interim finishes before the final message's chunks are enqueued.
|
|
interimBeforeFinal: lastInterimIdx >= 0 && firstFinalIdx >= 0 && lastInterimIdx < firstFinalIdx,
|
|
// (iv) the state never fell back to Thinking with an unspoken interim tail.
|
|
thinkingWithUnspokenTail: violations.length,
|
|
state: harness.state(),
|
|
};
|
|
};
|
|
|
|
// UNIFIED OUTPUT SINK + device selector. When the browser can enumerate outputs
|
|
// and setSinkId, a tidy corner control lists them; choosing one routes ALL spoken
|
|
// output (the reply's blob element here, and — via the shared AudioContext — the
|
|
// PCM reply and thinking cues) to that single device. When the APIs are missing,
|
|
// the control hides entirely (a dead control never appears).
|
|
scenarios.output_device_selector_routes_spoken_output = async () => {
|
|
const harness = makeHarness({
|
|
outputs: [
|
|
{ deviceId: '', kind: 'audiooutput', label: 'System default' },
|
|
{ deviceId: 'spk-1', kind: 'audiooutput', label: 'Speaker One' },
|
|
{ deviceId: 'spk-2', kind: 'audiooutput', label: 'Headphones Two' },
|
|
{ deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' },
|
|
],
|
|
});
|
|
await harness.start();
|
|
await harness.flush();
|
|
await harness.flush();
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const outWrap = overlay.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-out') >= 0,
|
|
);
|
|
const outBtn = outWrap ? outWrap.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-out-btn') >= 0,
|
|
) : null;
|
|
const outMenu = outWrap ? outWrap.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-out-menu') >= 0,
|
|
) : null;
|
|
const shownAfterRefresh = outWrap ? outWrap.style.display !== 'none' : false;
|
|
const items = outMenu ? outMenu.children : [];
|
|
const itemLabels = items.map((item) => item.textContent);
|
|
const headphones = items.find((item) => item.getAttribute('data-device') === 'spk-2');
|
|
if (headphones) headphones.click();
|
|
const forcedChecked = headphones ? headphones.getAttribute('aria-checked') : null;
|
|
const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false;
|
|
// A reply now plays through the blob fallback element, which must be routed to
|
|
// the chosen sink.
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
harness.setAssistantReply('A short spoken reply.');
|
|
harness.completeResponse();
|
|
await harness.tick(400);
|
|
const sinks = harness.sinkCalls();
|
|
return {
|
|
overlayPresent: true,
|
|
supported: true,
|
|
shownAfterRefresh,
|
|
itemLabels,
|
|
forcedChecked,
|
|
btnForced,
|
|
audioRoutedTo: sinks.filter((c) => c.kind === 'audio').map((c) => c.sink),
|
|
};
|
|
};
|
|
|
|
// When enumerateDevices/setSinkId are unavailable the selector is present in the
|
|
// DOM but hidden — never a dead control.
|
|
scenarios.output_selector_hidden_when_unsupported = async () => {
|
|
const harness = makeHarness();
|
|
await harness.start();
|
|
await harness.flush();
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const outWrap = overlay.children.find(
|
|
(child) => String(child.className).indexOf('voice-conversation-out') >= 0,
|
|
);
|
|
return {
|
|
overlayPresent: true,
|
|
controlInDom: !!outWrap,
|
|
hidden: outWrap ? outWrap.style.display === 'none' : null,
|
|
};
|
|
};
|
|
|
|
// Shape of the attached full-screen overlay: the essential orb + captions +
|
|
// mute/exit that MUST render whenever conversation mode opens, independent of the
|
|
// language/output corner selectors.
|
|
function overlayShape(harness) {
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const kids = overlay.children;
|
|
const find = (sub) => kids.find((c) => String(c.className).indexOf(sub) >= 0) || null;
|
|
const orb = find('voice-conversation-orb');
|
|
const orbMark = orb
|
|
? orb.children.find((c) => String(c.className).indexOf('voice-conversation-orb-mark') >= 0)
|
|
: null;
|
|
const captions = find('voice-conversation-captions');
|
|
const controls = find('voice-conversation-controls');
|
|
const mute = controls
|
|
? controls.children.find((c) => String(c.className).indexOf('voice-conversation-mute') >= 0)
|
|
: null;
|
|
const exit = controls
|
|
? controls.children.find((c) => String(c.className).indexOf('voice-conversation-exit') >= 0)
|
|
: null;
|
|
return {
|
|
overlayPresent: true,
|
|
isDialog: overlay.getAttribute('role') === 'dialog',
|
|
hasOrb: !!orb,
|
|
hasOrbMark: !!orbMark,
|
|
hasCaptions: !!captions,
|
|
captionCount: captions ? captions.children.length : 0,
|
|
hasMute: !!mute,
|
|
hasExit: !!exit,
|
|
};
|
|
}
|
|
|
|
// REGRESSION LOCK: the full-screen overlay must ALWAYS attach when conversation
|
|
// mode opens, even when the audio-output APIs are hostile — mediaDevices absent,
|
|
// or enumerateDevices rejecting. A failure here previously discarded the whole
|
|
// overlay and dropped back to the inline voice bar.
|
|
scenarios.overlay_opens_despite_hostile_output_apis = async () => {
|
|
const results = {};
|
|
|
|
// (A) navigator.mediaDevices is undefined when the overlay is built.
|
|
{
|
|
const harness = makeHarness();
|
|
await harness.flush();
|
|
const internals = harness.context.window.__atlasVoiceInternals;
|
|
harness.context.navigator.mediaDevices = undefined;
|
|
internals.openConversationOverlay();
|
|
results.mediaDevicesUndefined = overlayShape(harness);
|
|
const overlay = harness.overlay();
|
|
const outWrap = overlay
|
|
? overlay.children.find((c) => String(c.className).indexOf('voice-conversation-out') >= 0)
|
|
: null;
|
|
results.mediaDevicesUndefined.outputHidden = outWrap ? outWrap.style.display === 'none' : null;
|
|
internals.removeConversationOverlay();
|
|
}
|
|
|
|
// (B) enumerateDevices is present (routing LOOKS supported) but REJECTS.
|
|
{
|
|
const harness = makeHarness();
|
|
await harness.flush();
|
|
const internals = harness.context.window.__atlasVoiceInternals;
|
|
harness.context.navigator.mediaDevices.enumerateDevices = async () => {
|
|
throw new Error('enumerate blocked');
|
|
};
|
|
internals.openConversationOverlay();
|
|
// The overlay must be attached synchronously, before the async rejection.
|
|
const immediate = overlayShape(harness);
|
|
await harness.flush();
|
|
await harness.flush();
|
|
results.enumerateRejects = overlayShape(harness);
|
|
results.enumerateRejects.attachedBeforeReject = immediate.overlayPresent;
|
|
internals.removeConversationOverlay();
|
|
}
|
|
return results;
|
|
};
|
|
|
|
// FIX 2(a): with the mic open the OS routes the system default to the earpiece;
|
|
// the shared sink must auto-default to the LOUDSPEAKER (never "communications"),
|
|
// with no user interaction — and that default must reach real playback.
|
|
scenarios.output_defaults_to_loudspeaker = async () => {
|
|
const harness = makeHarness({
|
|
outputs: [
|
|
{ deviceId: '', kind: 'audiooutput', label: 'System default' },
|
|
{ deviceId: 'communications', kind: 'audiooutput', label: 'Communications' },
|
|
{ deviceId: 'ear-1', kind: 'audiooutput', label: 'Earpiece' },
|
|
{ deviceId: 'spk-1', kind: 'audiooutput', label: 'Speakerphone' },
|
|
{ deviceId: 'mic-1', kind: 'audioinput', label: 'Microphone' },
|
|
],
|
|
});
|
|
await harness.start();
|
|
await harness.flush();
|
|
await harness.flush();
|
|
const overlay = harness.overlay();
|
|
if (!overlay) return { overlayPresent: false };
|
|
const outWrap = overlay.children.find((c) => String(c.className).indexOf('voice-conversation-out') >= 0);
|
|
const outBtn = outWrap ? outWrap.children.find((c) => String(c.className).indexOf('voice-conversation-out-btn') >= 0) : null;
|
|
const outMenu = outWrap ? outWrap.children.find((c) => String(c.className).indexOf('voice-conversation-out-menu') >= 0) : null;
|
|
const items = outMenu ? outMenu.children : [];
|
|
const defaultChecked = items
|
|
.filter((i) => i.getAttribute('aria-checked') === 'true')
|
|
.map((i) => i.getAttribute('data-device'));
|
|
const btnForced = outBtn ? String(outBtn.className).indexOf('is-forced') >= 0 : false;
|
|
const btnLabel = outBtn ? outBtn.getAttribute('aria-label') : null;
|
|
// The default must reach real playback with NO user interaction: a reply now
|
|
// plays through the blob fallback and must route to the auto-selected speaker.
|
|
await harness.silence(300);
|
|
await harness.speak('alpha', 1300);
|
|
await harness.silence(1500);
|
|
await harness.tick(400);
|
|
harness.setAssistantReply('A short spoken reply.');
|
|
harness.completeResponse();
|
|
await harness.tick(400);
|
|
const audioRoutedTo = harness.sinkCalls().filter((c) => c.kind === 'audio').map((c) => c.sink);
|
|
return {
|
|
overlayPresent: true,
|
|
defaultChecked,
|
|
btnForced,
|
|
btnLabel,
|
|
audioRoutedTo,
|
|
};
|
|
};
|
|
|
|
// FIX 2(a) pure logic: the loudspeaker chooser prefers a labelled speaker, never
|
|
// the "communications" endpoint, falls back to the first concrete non-earpiece
|
|
// output, and returns '' (system default) when only an earpiece/comms exists.
|
|
scenarios.loudspeaker_selection_logic = async () => {
|
|
const harness = makeHarness();
|
|
await harness.flush();
|
|
const pick = harness.context.window.__atlasVoiceInternals.pickLoudspeakerSink;
|
|
return {
|
|
labelledSpeaker: pick([
|
|
{ deviceId: 'ear', kind: 'audiooutput', label: 'Earpiece' },
|
|
{ deviceId: 'spk', kind: 'audiooutput', label: 'Speakerphone' },
|
|
]),
|
|
skipsCommunications: pick([
|
|
{ deviceId: 'communications', kind: 'audiooutput', label: 'Communications' },
|
|
{ deviceId: 'spk', kind: 'audiooutput', label: 'Speaker' },
|
|
]),
|
|
concreteFallback: pick([
|
|
{ deviceId: 'default', kind: 'audiooutput', label: '' },
|
|
{ deviceId: 'dev-9', kind: 'audiooutput', label: '' },
|
|
]),
|
|
onlyEarpiece: pick([
|
|
{ deviceId: 'communications', kind: 'audiooutput', label: 'Communications' },
|
|
{ deviceId: 'ear', kind: 'audiooutput', label: 'Earpiece' },
|
|
]),
|
|
ignoresInputs: pick([
|
|
{ deviceId: 'mic', kind: 'audioinput', label: 'Speaker Mic' },
|
|
{ deviceId: 'spk', kind: 'audiooutput', label: 'Loudspeaker' },
|
|
]),
|
|
};
|
|
};
|
|
|
|
(async () => {
|
|
const output = {};
|
|
for (const name of Object.keys(scenarios)) {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
output[name] = await scenarios[name]();
|
|
}
|
|
process.stdout.write(JSON.stringify(output, null, 1));
|
|
})().catch((error) => {
|
|
process.stderr.write(String((error && error.stack) || error));
|
|
process.exit(1);
|
|
});
|