fe45d23eae
Hands-free voice mode had no language signal at all, so every spoken reply was synthesized with the English voice no matter what the user actually said. The multilingual Piper work (PR #26) added server-side routing for a "language" field but nothing ever sent one. Carry the language the private Jetson Whisper service already detects through to the TTS request for the reply that speech produced, and only for that reply. hermes-stt returns {text, model, language}, accepted only as a bare ISO-639 token; hermes_stt_client.py writes a <stem>.language sidecar next to the .txt transcript Hermes reads, leaving the local-command contract intact; the patched local-command envelope and /api/transcribe re-validate it and surface it; atlas-voice.js binds it to the voice-mode generation token and chat session, consumes it exactly once, and clears it on cancellation, restart, session change, empty transcript or transcription error; /api/tts honours it only from the fixed en/ru/es allow-list and otherwise sends English. A client "voice" field is never read at any hop, and typed messages, the manual read-aloud button, and any reply not produced by a spoken turn carry no trusted signal and stay on the English voice. The two WebUI-side and one agent-side edits are fail-closed replace_exact patches; both patch roots are now env-overridable so the contract can be verified offline without a GPU or an image build.
453 lines
14 KiB
JavaScript
453 lines
14 KiB
JavaScript
// Deterministic browser stub that drives dockerfiles/hermes-webui-atlas-voice.js
|
||
// through complete hands-free turns with no microphone, audio device or GPU.
|
||
//
|
||
// The script under test is an IIFE with no exported seams, so the only honest
|
||
// way to assert what reaches POST /api/tts is to run it against a fake DOM and
|
||
// fake clock and record the requests it actually makes. Usage:
|
||
//
|
||
// node atlas_voice_language_probe.js <path-to-atlas-voice.js>
|
||
//
|
||
// It prints one JSON object describing every scenario to stdout.
|
||
'use strict';
|
||
|
||
const fs = require('fs');
|
||
const vm = require('vm');
|
||
|
||
const SCRIPT_PATH = process.argv[2];
|
||
if (!SCRIPT_PATH) {
|
||
throw new Error('usage: atlas_voice_language_probe.js <atlas-voice.js>');
|
||
}
|
||
const SOURCE = fs.readFileSync(SCRIPT_PATH, 'utf8');
|
||
|
||
function flush() {
|
||
// Four macrotask hops drain the promise chains the script builds around
|
||
// fetch()/json()/blob()/play() without ever waiting on wall-clock time.
|
||
return new Promise((resolve) => {
|
||
let hops = 0;
|
||
(function hop() {
|
||
hops += 1;
|
||
if (hops > 12) {
|
||
resolve();
|
||
return;
|
||
}
|
||
setImmediate(hop);
|
||
})();
|
||
});
|
||
}
|
||
|
||
function makeElement(id) {
|
||
return {
|
||
id,
|
||
style: {},
|
||
dataset: {},
|
||
value: '',
|
||
textContent: '',
|
||
className: '',
|
||
classList: {
|
||
entries: new Set(),
|
||
add(name) { this.entries.add(name); },
|
||
remove(name) { this.entries.delete(name); },
|
||
contains(name) { return this.entries.has(name); },
|
||
},
|
||
listeners: [],
|
||
addEventListener(type, handler) { this.listeners.push({ type, handler }); },
|
||
removeEventListener(type, handler) {
|
||
this.listeners = this.listeners.filter((entry) => entry.handler !== handler);
|
||
},
|
||
click() {
|
||
const event = { preventDefault() {}, stopImmediatePropagation() {} };
|
||
this.listeners
|
||
.filter((entry) => entry.type === 'click')
|
||
.forEach((entry) => entry.handler(event));
|
||
},
|
||
querySelector() { return null; },
|
||
insertBefore() {},
|
||
appendChild() {},
|
||
};
|
||
}
|
||
|
||
function makeHarness() {
|
||
const clock = { now: 1000000 };
|
||
const timeouts = [];
|
||
const intervals = new Map();
|
||
let timerId = 1;
|
||
|
||
const ttsRequests = [];
|
||
const transcribeCalls = [];
|
||
const toasts = [];
|
||
const sends = [];
|
||
|
||
let capability = { ok: true, available: true, provider: 'local_command' };
|
||
let transcribeResponse = { ok: true, transcript: 'hello', language: 'en' };
|
||
let transcribeStatus = 200;
|
||
let assistantRows = [];
|
||
let loud = false;
|
||
let recorder = null;
|
||
|
||
const storage = new Map();
|
||
const elements = {};
|
||
['btnVoiceMode', 'voiceModeBar', 'voiceModeIndicator', 'voiceModeLabel', 'msg']
|
||
.forEach((id) => { elements[id] = makeElement(id); });
|
||
|
||
function MediaRecorder() {
|
||
this.state = 'recording';
|
||
this.ondataavailable = null;
|
||
this.onstop = null;
|
||
recorder = this;
|
||
}
|
||
MediaRecorder.prototype.start = function start() { this.state = 'recording'; };
|
||
MediaRecorder.prototype.stop = function stop() {
|
||
if (this.state === 'inactive') return;
|
||
this.state = 'inactive';
|
||
if (this.onstop) this.onstop();
|
||
};
|
||
MediaRecorder.isTypeSupported = function isTypeSupported() { return true; };
|
||
|
||
function AudioContext() {
|
||
this.createAnalyser = () => ({
|
||
fftSize: 2048,
|
||
getByteTimeDomainData(samples) {
|
||
for (let i = 0; i < samples.length; i += 1) {
|
||
samples[i] = loud ? (i % 2 ? 200 : 56) : 128;
|
||
}
|
||
},
|
||
});
|
||
this.createBiquadFilter = () => ({
|
||
type: '', frequency: { value: 0 }, Q: { value: 0 }, connect() {},
|
||
});
|
||
this.createMediaStreamSource = () => ({ connect() {} });
|
||
this.close = () => {};
|
||
}
|
||
|
||
function AudioElement() {
|
||
this.currentTime = 0;
|
||
this.onended = null;
|
||
this.onerror = null;
|
||
this.pause = () => {};
|
||
this.play = () => {
|
||
setImmediate(() => { if (this.onended) this.onended(); });
|
||
return Promise.resolve();
|
||
};
|
||
}
|
||
|
||
async function fetchStub(url, init) {
|
||
if (url === '/api/transcribe/capability') {
|
||
return { ok: true, status: 200, json: async () => capability };
|
||
}
|
||
if (url === '/api/transcribe') {
|
||
transcribeCalls.push({ body: init && init.body });
|
||
return {
|
||
ok: transcribeStatus < 400,
|
||
status: transcribeStatus,
|
||
json: async () => transcribeResponse,
|
||
};
|
||
}
|
||
if (url === '/api/tts') {
|
||
ttsRequests.push(JSON.parse(init.body));
|
||
return {
|
||
ok: true,
|
||
status: 200,
|
||
blob: async () => ({ synthetic: true }),
|
||
json: async () => ({}),
|
||
};
|
||
}
|
||
throw new Error(`unexpected fetch: ${url}`);
|
||
}
|
||
|
||
const context = {
|
||
console,
|
||
Uint8Array,
|
||
Promise,
|
||
Math,
|
||
JSON,
|
||
String,
|
||
Number,
|
||
Error,
|
||
parseInt,
|
||
isNaN,
|
||
Set,
|
||
Map,
|
||
Array,
|
||
Object,
|
||
Date: { now: () => clock.now },
|
||
Blob: function Blob(parts, options) { this.parts = parts; this.type = (options || {}).type || ''; },
|
||
File: function File(parts, name, options) {
|
||
this.parts = parts; this.name = name; this.type = (options || {}).type || '';
|
||
},
|
||
FormData: function FormData() { this.entries = []; this.append = (k, v) => this.entries.push([k, v]); },
|
||
URL: { createObjectURL: () => 'blob:atlas-test', revokeObjectURL() {} },
|
||
Audio: AudioElement,
|
||
MediaRecorder,
|
||
AudioContext,
|
||
fetch: fetchStub,
|
||
localStorage: {
|
||
getItem: (key) => (storage.has(key) ? storage.get(key) : null),
|
||
setItem: (key, value) => { storage.set(key, String(value)); },
|
||
removeItem: (key) => { storage.delete(key); },
|
||
},
|
||
navigator: {
|
||
mediaDevices: {
|
||
getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }),
|
||
getSupportedConstraints: () => ({}),
|
||
},
|
||
},
|
||
document: {
|
||
getElementById: (id) => elements[id] || null,
|
||
querySelectorAll: () => assistantRows,
|
||
createElement: () => ({ value: '', textContent: '' }),
|
||
},
|
||
S: { session: { session_id: 'session-1' }, busy: false },
|
||
setTimeout: (fn, delay) => {
|
||
const id = timerId; timerId += 1;
|
||
timeouts.push({ id, fn, at: clock.now + (delay || 0) });
|
||
return id;
|
||
},
|
||
clearTimeout: (id) => {
|
||
const index = timeouts.findIndex((entry) => entry.id === id);
|
||
if (index >= 0) timeouts.splice(index, 1);
|
||
},
|
||
setInterval: (fn, delay) => {
|
||
const id = timerId; timerId += 1;
|
||
intervals.set(id, { fn, delay: delay || 0 });
|
||
return id;
|
||
},
|
||
clearInterval: (id) => { intervals.delete(id); },
|
||
};
|
||
context.window = context;
|
||
context.showToast = (message) => { toasts.push(message); };
|
||
context.send = () => { sends.push(elements.msg.value); };
|
||
context.autoResize = () => {};
|
||
|
||
vm.createContext(context);
|
||
vm.runInContext(SOURCE, context, { filename: 'atlas-voice.js' });
|
||
|
||
function runDueTimeouts() {
|
||
const due = timeouts.filter((entry) => entry.at <= clock.now);
|
||
due.forEach((entry) => {
|
||
const index = timeouts.indexOf(entry);
|
||
if (index >= 0) timeouts.splice(index, 1);
|
||
entry.fn();
|
||
});
|
||
}
|
||
|
||
function tick(ms) {
|
||
clock.now += ms;
|
||
Array.from(intervals.values()).forEach((entry) => entry.fn());
|
||
runDueTimeouts();
|
||
}
|
||
|
||
return {
|
||
context,
|
||
elements,
|
||
ttsRequests,
|
||
transcribeCalls,
|
||
toasts,
|
||
sends,
|
||
clock,
|
||
recorder: () => recorder,
|
||
setLoud: (value) => { loud = value; },
|
||
setCapability: (value) => { capability = value; },
|
||
setTranscribeResponse: (value, status) => {
|
||
transcribeResponse = value;
|
||
transcribeStatus = status === undefined ? 200 : status;
|
||
},
|
||
setAssistantReply: (text) => { assistantRows = [{ dataset: { rawText: text } }]; },
|
||
setSession: (id) => { context.S.session = { session_id: id }; },
|
||
advance: (ms) => { clock.now += ms; },
|
||
tick,
|
||
runDueTimeouts,
|
||
flush,
|
||
};
|
||
}
|
||
|
||
// Walk one capture window: pre-roll audio, three loud frames so the VAD latches
|
||
// speech, then silence past the hangover so MediaRecorder.stop() fires.
|
||
async function captureSpeech(harness) {
|
||
const active = harness.recorder();
|
||
if (!active) throw new Error('voice mode never created a recorder');
|
||
active.ondataavailable({ data: { size: 512 } });
|
||
harness.setLoud(true);
|
||
for (let i = 0; i < 4; i += 1) harness.tick(100);
|
||
active.ondataavailable({ data: { size: 512 } });
|
||
harness.setLoud(false);
|
||
harness.advance(2500);
|
||
harness.tick(100);
|
||
await harness.flush();
|
||
}
|
||
|
||
async function startVoiceMode(harness) {
|
||
await harness.flush();
|
||
harness.elements.btnVoiceMode.click();
|
||
await harness.flush();
|
||
}
|
||
|
||
// One complete turn: speak, transcribe, let the app "answer", read it back.
|
||
async function runTurn(harness, { transcript, language, reply }) {
|
||
const payload = { ok: true, transcript };
|
||
if (language !== undefined) payload.language = language;
|
||
harness.setTranscribeResponse(payload);
|
||
await captureSpeech(harness);
|
||
harness.setAssistantReply(reply || 'An answer.');
|
||
harness.context.autoReadLastAssistant();
|
||
await harness.flush();
|
||
}
|
||
|
||
async function restartListening(harness) {
|
||
harness.advance(1000);
|
||
harness.runDueTimeouts();
|
||
await harness.flush();
|
||
}
|
||
|
||
const scenarios = {};
|
||
|
||
scenarios.english_turn_speaks_english = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'What is the weather?', language: 'en' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.russian_turn_speaks_russian = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'Как дела?', language: 'ru', reply: 'Всё хорошо.' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.spanish_turn_speaks_spanish = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: '¿Qué tal?', language: 'es', reply: 'Muy bien.' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.missing_language_falls_back = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'Hello there.' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.unsupported_language_falls_back = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'Bonjour tout le monde.', language: 'fr' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.hostile_language_values_are_dropped = async () => {
|
||
const results = [];
|
||
const hostile = [
|
||
'ru; rm -rf /',
|
||
'../../ru_RU-irina-medium',
|
||
'ru ',
|
||
'RUSSIAN',
|
||
{ language: 'ru' },
|
||
['ru'],
|
||
42,
|
||
null,
|
||
'r',
|
||
'ru ru',
|
||
'x'.repeat(4096),
|
||
];
|
||
for (const language of hostile) {
|
||
const harness = makeHarness();
|
||
// eslint-disable-next-line no-await-in-loop
|
||
await startVoiceMode(harness);
|
||
// eslint-disable-next-line no-await-in-loop
|
||
await runTurn(harness, { transcript: 'Say something.', language });
|
||
results.push({ sent: String(language), tts: harness.ttsRequests });
|
||
}
|
||
return { results };
|
||
};
|
||
|
||
scenarios.voice_field_is_never_sent = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'Как дела?', language: 'ru' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.language_does_not_leak_into_later_turn = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
await runTurn(harness, { transcript: 'Как дела?', language: 'ru', reply: 'Всё хорошо.' });
|
||
await restartListening(harness);
|
||
await runTurn(harness, { transcript: 'And in English?', language: undefined });
|
||
await restartListening(harness);
|
||
await runTurn(harness, { transcript: '¿Y ahora?', language: 'es' });
|
||
return { tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.empty_transcript_does_not_arm_a_language = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
harness.setTranscribeResponse({ ok: true, transcript: ' ', language: 'ru' });
|
||
await captureSpeech(harness);
|
||
const sendsAfterBlank = harness.sends.slice();
|
||
// A reply landing while the blank turn winds down must not inherit a
|
||
// language that transcript never earned.
|
||
harness.setAssistantReply('A stray answer.');
|
||
harness.context.autoReadLastAssistant();
|
||
await harness.flush();
|
||
await restartListening(harness);
|
||
await runTurn(harness, { transcript: 'Hello.', language: undefined });
|
||
return { sendsAfterBlank, tts: harness.ttsRequests, sends: harness.sends };
|
||
};
|
||
|
||
scenarios.session_change_discards_language = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
harness.setTranscribeResponse({ ok: true, transcript: 'Как дела?', language: 'ru' });
|
||
await captureSpeech(harness);
|
||
harness.setSession('session-2');
|
||
harness.setAssistantReply('Reply that belongs to another chat.');
|
||
harness.context.autoReadLastAssistant();
|
||
await harness.flush();
|
||
const afterSwitch = harness.ttsRequests.slice();
|
||
await restartListening(harness);
|
||
await runTurn(harness, { transcript: 'Hello again.', language: undefined });
|
||
return { afterSwitch, tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.deactivation_discards_language = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
harness.setTranscribeResponse({ ok: true, transcript: 'Как дела?', language: 'ru' });
|
||
await captureSpeech(harness);
|
||
harness.elements.btnVoiceMode.click();
|
||
await harness.flush();
|
||
harness.setAssistantReply('Late reply after the user left voice mode.');
|
||
harness.context.autoReadLastAssistant();
|
||
await harness.flush();
|
||
const afterDeactivate = harness.ttsRequests.slice();
|
||
harness.elements.btnVoiceMode.click();
|
||
await harness.flush();
|
||
await runTurn(harness, { transcript: 'Fresh start.', language: undefined });
|
||
return { afterDeactivate, tts: harness.ttsRequests };
|
||
};
|
||
|
||
scenarios.transcribe_error_speaks_nothing = async () => {
|
||
const harness = makeHarness();
|
||
await startVoiceMode(harness);
|
||
harness.setTranscribeResponse({ error: 'Whisper is down' }, 503);
|
||
await captureSpeech(harness);
|
||
harness.setAssistantReply('Some earlier answer.');
|
||
harness.context.autoReadLastAssistant();
|
||
await harness.flush();
|
||
return { tts: harness.ttsRequests, toasts: harness.toasts };
|
||
};
|
||
|
||
(async () => {
|
||
const output = {};
|
||
const names = Object.keys(scenarios);
|
||
for (const name of names) {
|
||
// eslint-disable-next-line no-await-in-loop
|
||
output[name] = await scenarios[name]();
|
||
}
|
||
process.stdout.write(JSON.stringify(output, null, 1));
|
||
})().catch((error) => {
|
||
process.stderr.write(String((error && error.stack) || error));
|
||
process.exit(1);
|
||
});
|