Fix hands-free STT WebM conversion #43
@ -90,12 +90,14 @@ PY
|
||||
# Add the Atlas voice bridge as a narrow integration layer. It activates only
|
||||
# when a tenant's server-side STT capability reports the private Jetson route.
|
||||
COPY dockerfiles/hermes-webui-atlas-patch.py /tmp/hermes-webui-atlas-patch.py
|
||||
COPY dockerfiles/hermes-webui-stt-patch.py /tmp/hermes-webui-stt-patch.py
|
||||
COPY dockerfiles/hermes-webui-telegram-project-patch.py /tmp/hermes-webui-telegram-project-patch.py
|
||||
COPY dockerfiles/hermes-webui-atlas-voice.js /opt/hermes-webui/static/atlas-voice.js
|
||||
COPY dockerfiles/hermes-webui-atlas-voice.css /opt/hermes-webui/static/atlas-voice.css
|
||||
COPY dockerfiles/hermes-webui-router-patch.py /tmp/hermes-webui-router-patch.py
|
||||
COPY dockerfiles/hermes-webui-router.js /opt/hermes-webui/static/atlas-router.js
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-atlas-patch.py
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-stt-patch.py
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-telegram-project-patch.py
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-router-patch.py
|
||||
|
||||
@ -109,6 +111,7 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
|
||||
&& grep -Fq "'atlas/auto/maximum': 'Automatic · Maximum'" /opt/hermes-webui/static/panels.js \
|
||||
&& grep -Fq 'Atlas Jetson (private)' /opt/hermes-webui/static/index.html \
|
||||
&& grep -Fq 'HERMES_WEBUI_ATLAS_TTS_URL' /opt/hermes-webui/api/routes.py \
|
||||
&& grep -Fq 'Audio conversion failed: upload is invalid' /opt/hermes/tools/transcription_tools.py \
|
||||
&& grep -Fq "capability.provider!=='local_command'" /opt/hermes-webui/static/atlas-voice.js \
|
||||
&& grep -Fq 'prefers-reduced-motion: reduce' /opt/hermes-webui/static/atlas-voice.css \
|
||||
&& grep -Fq 'id="voiceInstrumentStyles"' /opt/hermes-webui/static/index.html \
|
||||
@ -118,7 +121,8 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
|
||||
&& grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \
|
||||
&& /opt/hermes/.venv/bin/python -m py_compile \
|
||||
/opt/hermes-webui/api/routes.py \
|
||||
/opt/hermes-webui/api/gateway_chat.py
|
||||
/opt/hermes-webui/api/gateway_chat.py \
|
||||
/opt/hermes/tools/transcription_tools.py
|
||||
|
||||
# Exercise the real server process in the target architecture before publish.
|
||||
RUN set -eu; \
|
||||
|
||||
@ -131,10 +131,17 @@
|
||||
if(typeof window.send==='function') window.send();
|
||||
}
|
||||
|
||||
function audioExtension(mimeType){
|
||||
const normalized=String(mimeType||'').toLowerCase();
|
||||
if(normalized.indexOf('ogg')>=0) return 'ogg';
|
||||
if(normalized.indexOf('mp4')>=0) return 'mp4';
|
||||
return 'webm';
|
||||
}
|
||||
|
||||
async function transcribe(blob, token){
|
||||
if(!active||token!==generation) return;
|
||||
setState('transcribing');
|
||||
const ext=(blob.type||'').indexOf('ogg')>=0?'ogg':'webm';
|
||||
const ext=audioExtension(blob.type);
|
||||
const form=new FormData();
|
||||
form.append('file',new File([blob],'voice-input.'+ext,{type:blob.type||'audio/'+ext}));
|
||||
try{
|
||||
@ -182,19 +189,26 @@
|
||||
audioContext.createMediaStreamSource(stream).connect(highpass);
|
||||
highpass.connect(analyser);
|
||||
const samples=new Uint8Array(analyser.fftSize);
|
||||
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/webm'];
|
||||
const mimeTypes=['audio/webm;codecs=opus','audio/ogg;codecs=opus','audio/mp4;codecs=mp4a.40.2','audio/mp4','audio/webm'];
|
||||
const mime=mimeTypes.find(function(value){return MediaRecorder.isTypeSupported(value);})||'';
|
||||
const chunks=[];
|
||||
const preRoll=[];
|
||||
let initialChunk=null;
|
||||
let heardSpeech=false;
|
||||
let voiceFrames=0;
|
||||
let noiseFloor=0.008;
|
||||
let lastSpeech=Date.now();
|
||||
const started=Date.now();
|
||||
recorder=new MediaRecorder(stream,mime?{mimeType:mime}:undefined);
|
||||
let recordedMime=recorder.mimeType||mime||'';
|
||||
recorder.ondataavailable=function(event){
|
||||
if(!event.data||!event.data.size) return;
|
||||
if(event.data.type) recordedMime=event.data.type;
|
||||
if(heardSpeech){chunks.push(event.data);return;}
|
||||
// MediaRecorder's first timeslice owns the container initialization
|
||||
// (EBML/Opus headers for WebM, and equivalent headers for Ogg/MP4).
|
||||
// Keep it separately while bounding the actual audio pre-roll.
|
||||
if(!initialChunk){initialChunk=event.data;return;}
|
||||
preRoll.push(event.data);
|
||||
while(preRoll.length>3) preRoll.shift();
|
||||
};
|
||||
@ -207,7 +221,7 @@
|
||||
recorder=null;
|
||||
if(!active||token!==generation) return;
|
||||
if(!heardSpeech||!chunks.length){restartSoon(token,300);return;}
|
||||
transcribe(new Blob(chunks,{type:mime||'audio/webm'}),token);
|
||||
transcribe(new Blob(chunks,{type:recordedMime||'audio/webm'}),token);
|
||||
};
|
||||
recorder.start(250);
|
||||
const silenceMs=Math.max(900,parseInt(localStorage.getItem('hermes-voice-silence-ms')||'1600',10)||1600);
|
||||
@ -229,6 +243,7 @@
|
||||
if(!heardSpeech&&voiceFrames>=3){
|
||||
heardSpeech=true;
|
||||
lastSpeech=now;
|
||||
if(initialChunk){chunks.push(initialChunk);initialChunk=null;}
|
||||
while(preRoll.length) chunks.push(preRoll.shift());
|
||||
}else if(heardSpeech&&voiceNow){
|
||||
lastSpeech=now;
|
||||
|
||||
37
dockerfiles/hermes-webui-stt-patch.py
Normal file
37
dockerfiles/hermes-webui-stt-patch.py
Normal file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bound local-STT conversion errors in the pinned Hermes Agent source."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(os.environ.get("HERMES_AGENT_PATCH_ROOT", "/opt/hermes"))
|
||||
|
||||
|
||||
def replace_exact(path: Path, before: str, after: str) -> None:
|
||||
"""Replace one exact upstream fragment, failing closed on image drift."""
|
||||
source = path.read_text(encoding="utf-8")
|
||||
if source.count(before) != 1:
|
||||
raise SystemExit(f"Hermes STT patch context changed in {path}: {before[:80]!r}")
|
||||
path.write_text(source.replace(before, after, 1), encoding="utf-8")
|
||||
|
||||
|
||||
transcription = ROOT / "tools/transcription_tools.py"
|
||||
replace_exact(
|
||||
transcription,
|
||||
""" except subprocess.CalledProcessError as e:
|
||||
details = e.stderr.strip() or e.stdout.strip() or str(e)
|
||||
logger.error("ffmpeg conversion failed for %s: %s", file_path, details)
|
||||
return None, f"Failed to convert audio for local STT: {details}"
|
||||
""",
|
||||
""" except subprocess.CalledProcessError as e:
|
||||
details = e.stderr.strip() or e.stdout.strip() or str(e)
|
||||
logger.error(
|
||||
"ffmpeg conversion failed for %s: %s", file_path, details[-2000:]
|
||||
)
|
||||
return None, (
|
||||
"Audio conversion failed: upload is invalid, incomplete, or uses an "
|
||||
"unsupported codec"
|
||||
)
|
||||
""",
|
||||
)
|
||||
12
testing/fixtures/hermes-agent-9de9c25f/SOURCE.md
Normal file
12
testing/fixtures/hermes-agent-9de9c25f/SOURCE.md
Normal file
@ -0,0 +1,12 @@
|
||||
# Hermes Agent STT fixture provenance
|
||||
|
||||
This minimal module contains the exact `_prepare_local_audio` patch context
|
||||
extracted from the agent image inherited by `Dockerfile.hermes-webui`:
|
||||
|
||||
- Image: `registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107`
|
||||
- OCI platform manifest: `sha256:9b4c00a25fd08f0df3bcb800d755bd48576fb65af5ab0df40058fbb78a147e43`
|
||||
- Upstream Hermes revision: `9de9c25f620ff7f1ce0fd5457d596052d5159596`
|
||||
- Full upstream `tools/transcription_tools.py` SHA-256: `d7d7df56b98dfadc0a7c08d5addd02db4e3106e6fd3b753e9241c23145eb2951`
|
||||
|
||||
Tests execute the production patcher against this source fragment and then run
|
||||
its real ffmpeg conversion boundary with browser-produced bytes.
|
||||
@ -0,0 +1,44 @@
|
||||
"""Exact pinned local-audio preparation fragment with its direct dependencies."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
|
||||
LOCAL_NATIVE_AUDIO_FORMATS = {".wav", ".aiff", ".aif"}
|
||||
logger = __import__("logging").getLogger(__name__)
|
||||
|
||||
|
||||
def _find_ffmpeg_binary() -> Optional[str]:
|
||||
return shutil.which("ffmpeg")
|
||||
|
||||
|
||||
def windows_hide_flags() -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _prepare_local_audio(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Normalize audio for local CLI STT when needed."""
|
||||
audio_path = Path(file_path)
|
||||
if audio_path.suffix.lower() in LOCAL_NATIVE_AUDIO_FORMATS:
|
||||
return file_path, None
|
||||
|
||||
ffmpeg = _find_ffmpeg_binary()
|
||||
if not ffmpeg:
|
||||
return None, "Local STT fallback requires ffmpeg for non-WAV inputs, but ffmpeg was not found"
|
||||
|
||||
converted_path = os.path.join(work_dir, f"{audio_path.stem}.wav")
|
||||
command = [ffmpeg, "-y", "-i", file_path, converted_path]
|
||||
|
||||
try:
|
||||
subprocess.run(command, check=True, capture_output=True, text=True, timeout=300, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags())
|
||||
return converted_path, None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("ffmpeg conversion timed out for %s", file_path)
|
||||
return None, "Audio conversion for local STT timed out"
|
||||
except subprocess.CalledProcessError as e:
|
||||
details = e.stderr.strip() or e.stdout.strip() or str(e)
|
||||
logger.error("ffmpeg conversion failed for %s: %s", file_path, details)
|
||||
return None, f"Failed to convert audio for local STT: {details}"
|
||||
18
testing/fixtures/mediarecorder/SOURCE.md
Normal file
18
testing/fixtures/mediarecorder/SOURCE.md
Normal file
@ -0,0 +1,18 @@
|
||||
# Chromium MediaRecorder fixture provenance
|
||||
|
||||
`chromium-webm-opus.webm` is the concatenation of ten successive 250 ms-ish
|
||||
data blobs emitted by a real Chromium `MediaRecorder` for
|
||||
`audio/webm;codecs=opus`; `chromium-webm-opus.json` records their exact byte
|
||||
lengths so tests can replay each browser event. The recording has leading
|
||||
silence followed by a 440 Hz signal, mirroring speech that begins after the
|
||||
hands-free VAD pre-roll is full.
|
||||
|
||||
- Browser: Chromium 140.0.7339.16 (Playwright build v1187, arm64)
|
||||
- Generator: `testing/probes/generate_mediarecorder_fixture.js`
|
||||
- Timeslice request: 250 ms
|
||||
- Complete concatenated payload: WebM/Opus, accepted by ffmpeg 7.1.5
|
||||
- Complete payload SHA-256: `5585014ecad185511a727ccf47aa8b1ac70ea6f3efd7778309742bf88e6faf51`
|
||||
- First five chunks are treated as pre-speech by the test model; the production
|
||||
pre-roll retains only the most recent three.
|
||||
|
||||
The fixture is generated data, not recorded speech.
|
||||
20
testing/fixtures/mediarecorder/chromium-webm-opus.json
Normal file
20
testing/fixtures/mediarecorder/chromium-webm-opus.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"mime_type": "audio/webm;codecs=opus",
|
||||
"timeslice_ms": 250,
|
||||
"pre_speech_chunk_count": 5,
|
||||
"chunk_sizes": [
|
||||
204,
|
||||
70,
|
||||
70,
|
||||
70,
|
||||
70,
|
||||
3864,
|
||||
5796,
|
||||
4830,
|
||||
4830,
|
||||
1945
|
||||
],
|
||||
"payload_file": "chromium-webm-opus.webm",
|
||||
"browser": "140.0.7339.16",
|
||||
"generator": "testing/probes/generate_mediarecorder_fixture.js"
|
||||
}
|
||||
BIN
testing/fixtures/mediarecorder/chromium-webm-opus.webm
Normal file
BIN
testing/fixtures/mediarecorder/chromium-webm-opus.webm
Normal file
Binary file not shown.
84
testing/probes/generate_mediarecorder_fixture.js
Normal file
84
testing/probes/generate_mediarecorder_fixture.js
Normal file
@ -0,0 +1,84 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { chromium } = require('playwright');
|
||||
|
||||
async function main() {
|
||||
const outputPath = process.argv[2];
|
||||
if (!outputPath) throw new Error('usage: node generate_mediarecorder_fixture.js OUTPUT.json');
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--autoplay-policy=no-user-gesture-required'],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
const fixture = await page.evaluate(async () => {
|
||||
const mimeType = 'audio/webm;codecs=opus';
|
||||
if (!MediaRecorder.isTypeSupported(mimeType)) {
|
||||
throw new Error(`${mimeType} is not supported by this Chromium build`);
|
||||
}
|
||||
|
||||
const context = new AudioContext({sampleRate: 48000});
|
||||
const destination = context.createMediaStreamDestination();
|
||||
const oscillator = context.createOscillator();
|
||||
const gain = context.createGain();
|
||||
oscillator.frequency.value = 440;
|
||||
gain.gain.value = 0;
|
||||
oscillator.connect(gain).connect(destination);
|
||||
oscillator.start();
|
||||
|
||||
const chunks = [];
|
||||
const recorder = new MediaRecorder(destination.stream, {mimeType});
|
||||
recorder.ondataavailable = event => {
|
||||
if (event.data && event.data.size) chunks.push(event.data);
|
||||
};
|
||||
const stopped = new Promise(resolve => { recorder.onstop = resolve; });
|
||||
const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||
|
||||
recorder.start(250);
|
||||
await delay(1400);
|
||||
gain.gain.setValueAtTime(0.35, context.currentTime);
|
||||
await delay(900);
|
||||
gain.gain.setValueAtTime(0, context.currentTime);
|
||||
await delay(500);
|
||||
recorder.stop();
|
||||
await stopped;
|
||||
|
||||
oscillator.stop();
|
||||
await context.close();
|
||||
const encoded = [];
|
||||
for (const chunk of chunks) {
|
||||
const bytes = new Uint8Array(await chunk.arrayBuffer());
|
||||
let binary = '';
|
||||
for (let index = 0; index < bytes.length; index += 0x8000) {
|
||||
binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000));
|
||||
}
|
||||
encoded.push(btoa(binary));
|
||||
}
|
||||
return {
|
||||
mime_type: recorder.mimeType,
|
||||
timeslice_ms: 250,
|
||||
pre_speech_chunk_count: 5,
|
||||
chunks_base64: encoded,
|
||||
};
|
||||
});
|
||||
const chunks = fixture.chunks_base64.map(value => Buffer.from(value, 'base64'));
|
||||
const payloadPath = outputPath.replace(/\.json$/i, '.webm');
|
||||
fixture.chunk_sizes = chunks.map(chunk => chunk.length);
|
||||
fixture.payload_file = path.basename(payloadPath);
|
||||
delete fixture.chunks_base64;
|
||||
fixture.browser = await browser.version();
|
||||
fixture.generator = 'testing/probes/generate_mediarecorder_fixture.js';
|
||||
fs.writeFileSync(payloadPath, Buffer.concat(chunks));
|
||||
fs.writeFileSync(outputPath, `${JSON.stringify(fixture, null, 2)}\n`);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
|
||||
class StyleDeclaration {
|
||||
@ -92,6 +93,7 @@ async function boot(scriptPath, reduced) {
|
||||
const reducedMotion = {matches: reduced, addEventListener() {}, removeEventListener() {}};
|
||||
let rejectNextCapture = false;
|
||||
let transcriptResolve;
|
||||
let uploadedFile = null;
|
||||
let sent = 0;
|
||||
let lastAudio = null;
|
||||
|
||||
@ -117,9 +119,10 @@ async function boot(scriptPath, reduced) {
|
||||
}
|
||||
class FakeMediaRecorder {
|
||||
static isTypeSupported() { return true; }
|
||||
constructor(stream) {
|
||||
constructor(stream, options) {
|
||||
this.stream = stream;
|
||||
this.state = 'inactive';
|
||||
this.mimeType = options?.mimeType || 'audio/webm;codecs=opus';
|
||||
this.ondataavailable = null;
|
||||
this.onstop = null;
|
||||
recorders.push(this);
|
||||
@ -171,11 +174,18 @@ async function boot(scriptPath, reduced) {
|
||||
},
|
||||
},
|
||||
};
|
||||
async function fetch(url) {
|
||||
async function fetch(url, options) {
|
||||
if (url === '/api/transcribe/capability') {
|
||||
return {ok: true, json: async () => ({available: true, provider: 'local_command'})};
|
||||
}
|
||||
if (url === '/api/transcribe') {
|
||||
const file = options?.body?.get('file');
|
||||
assert.ok(file, 'transcription request did not carry a file');
|
||||
uploadedFile = {
|
||||
bytes: new Uint8Array(await file.arrayBuffer()),
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
};
|
||||
return new Promise(resolve => { transcriptResolve = resolve; });
|
||||
}
|
||||
if (url === '/api/tts') {
|
||||
@ -245,6 +255,7 @@ async function boot(scriptPath, reduced) {
|
||||
captures, recorders, analysers, assistantRows, toasts, reducedMotion,
|
||||
get sent() { return sent; },
|
||||
get lastAudio() { return lastAudio; },
|
||||
get uploadedFile() { return uploadedFile; },
|
||||
set now(value) { now = value; },
|
||||
rejectCapture() { rejectNextCapture = true; },
|
||||
resolveTranscript(payload) {
|
||||
@ -261,7 +272,7 @@ async function boot(scriptPath, reduced) {
|
||||
};
|
||||
}
|
||||
|
||||
async function normalMotionContract(scriptPath) {
|
||||
async function normalMotionContract(scriptPath, mediaFixture) {
|
||||
const probe = await boot(scriptPath, false);
|
||||
const originalStyleLink = probe.styleLink;
|
||||
|
||||
@ -272,17 +283,38 @@ async function normalMotionContract(scriptPath) {
|
||||
await flush();
|
||||
assert.equal(probe.captures.length, 1);
|
||||
|
||||
const mediaChunks = mediaFixture.chunks;
|
||||
const preSpeechCount = mediaFixture.pre_speech_chunk_count;
|
||||
for (const chunk of mediaChunks.slice(0, preSpeechCount)) {
|
||||
probe.recorders[0].ondataavailable({
|
||||
data: new Blob([chunk], {type: mediaFixture.mime_type}),
|
||||
});
|
||||
}
|
||||
|
||||
probe.analysers[0].level = 0.3;
|
||||
probe.runIntervals();
|
||||
probe.runIntervals();
|
||||
probe.runIntervals();
|
||||
probe.recorders[0].ondataavailable({data: new Blob(['speech'], {type: 'audio/webm'})});
|
||||
for (const chunk of mediaChunks.slice(preSpeechCount)) {
|
||||
probe.recorders[0].ondataavailable({
|
||||
data: new Blob([chunk], {type: mediaFixture.mime_type}),
|
||||
});
|
||||
}
|
||||
probe.now = 4000;
|
||||
probe.analysers[0].level = 0;
|
||||
probe.runIntervals();
|
||||
await flush();
|
||||
await flush();
|
||||
assert.match(probe.indicator.className, /\btranscribing\b/);
|
||||
assert.equal(probe.label.textContent, 'Transcribing…');
|
||||
assert.ok(probe.uploadedFile, 'transcription upload was not captured');
|
||||
assert.equal(probe.uploadedFile.name, 'voice-input.webm');
|
||||
assert.equal(probe.uploadedFile.type, mediaFixture.mime_type);
|
||||
const expectedUpload = Buffer.concat([
|
||||
mediaChunks[0],
|
||||
...mediaChunks.slice(preSpeechCount - 3),
|
||||
]);
|
||||
assert.deepEqual(Buffer.from(probe.uploadedFile.bytes), expectedUpload);
|
||||
|
||||
probe.resolveTranscript({transcript: 'Hello Hermes'});
|
||||
await flush();
|
||||
@ -310,8 +342,21 @@ async function normalMotionContract(scriptPath) {
|
||||
|
||||
async function main() {
|
||||
const scriptPath = process.argv[2];
|
||||
assert.ok(scriptPath, 'usage: node hermes_voice_instrument_probe.js <atlas-voice.js>');
|
||||
const first = await normalMotionContract(scriptPath);
|
||||
const fixturePath = process.argv[3];
|
||||
assert.ok(
|
||||
scriptPath && fixturePath,
|
||||
'usage: node hermes_voice_instrument_probe.js <atlas-voice.js> <media-fixture.json>',
|
||||
);
|
||||
const mediaFixture = JSON.parse(fs.readFileSync(fixturePath, 'utf8'));
|
||||
const payload = fs.readFileSync(path.join(path.dirname(fixturePath), mediaFixture.payload_file));
|
||||
let offset = 0;
|
||||
mediaFixture.chunks = mediaFixture.chunk_sizes.map(size => {
|
||||
const chunk = payload.subarray(offset, offset + size);
|
||||
offset += size;
|
||||
return chunk;
|
||||
});
|
||||
assert.equal(offset, payload.length, 'fixture chunk sizes do not cover the payload');
|
||||
const first = await normalMotionContract(scriptPath, mediaFixture);
|
||||
const {probe, originalStyleLink, capturesBeforeSpeech} = first;
|
||||
|
||||
assert.equal(probe.captures.length, capturesBeforeSpeech);
|
||||
|
||||
203
testing/tests/test_hermes_handsfree_stt.py
Normal file
203
testing/tests/test_hermes_handsfree_stt.py
Normal file
@ -0,0 +1,203 @@
|
||||
"""Browser-to-ffmpeg contracts for Hermes hands-free transcription uploads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js"
|
||||
STT_PATCHER = ROOT / "dockerfiles/hermes-webui-stt-patch.py"
|
||||
AGENT_FIXTURE = ROOT / "testing/fixtures/hermes-agent-9de9c25f"
|
||||
MEDIA_FIXTURE = ROOT / "testing/fixtures/mediarecorder/chromium-webm-opus.json"
|
||||
SAFE_CONVERSION_ERROR = (
|
||||
"Audio conversion failed: upload is invalid, incomplete, or uses an "
|
||||
"unsupported codec"
|
||||
)
|
||||
|
||||
|
||||
def _media_chunks() -> tuple[dict[str, object], list[bytes]]:
|
||||
fixture = json.loads(MEDIA_FIXTURE.read_text(encoding="utf-8"))
|
||||
payload = MEDIA_FIXTURE.with_name(str(fixture["payload_file"])).read_bytes()
|
||||
chunks = []
|
||||
offset = 0
|
||||
for size in fixture["chunk_sizes"]:
|
||||
end = offset + int(size)
|
||||
chunks.append(payload[offset:end])
|
||||
offset = end
|
||||
assert offset == len(payload)
|
||||
return fixture, chunks
|
||||
|
||||
|
||||
def _late_speech_upload(*, preserve_header: bool) -> bytes:
|
||||
fixture, chunks = _media_chunks()
|
||||
count = int(fixture["pre_speech_chunk_count"])
|
||||
retained = chunks[count - 3 :]
|
||||
if preserve_header:
|
||||
retained.insert(0, chunks[0])
|
||||
return b"".join(retained)
|
||||
|
||||
|
||||
def _patched_transcription_module(tmp_path: Path):
|
||||
target = tmp_path / "hermes-agent"
|
||||
shutil.copytree(AGENT_FIXTURE, target)
|
||||
env = os.environ.copy()
|
||||
env["HERMES_AGENT_PATCH_ROOT"] = str(target)
|
||||
subprocess.run(
|
||||
[sys.executable, str(STT_PATCHER)],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
module_path = target / "tools/transcription_tools.py"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"pinned_transcription_tools", module_path
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _convert(module, tmp_path: Path, payload: bytes, suffix: str):
|
||||
source = tmp_path / f"voice-input{suffix}"
|
||||
output_dir = tmp_path / f"converted-{suffix.removeprefix('.')}"
|
||||
output_dir.mkdir()
|
||||
source.write_bytes(payload)
|
||||
return module._prepare_local_audio(str(source), str(output_dir))
|
||||
|
||||
|
||||
def test_fixture_reproduces_the_current_ebml_header_failure(tmp_path: Path):
|
||||
"""The red control is browser data after the old sliding pre-roll dropped chunk 0."""
|
||||
broken = tmp_path / "current-late-speech.webm"
|
||||
broken.write_bytes(_late_speech_upload(preserve_header=False))
|
||||
|
||||
result = subprocess.run(
|
||||
["ffmpeg", "-v", "error", "-y", "-i", str(broken), str(tmp_path / "bad.wav")],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "EBML header" in result.stderr
|
||||
|
||||
|
||||
def test_header_preserved_browser_webm_reaches_real_conversion_boundary(
|
||||
tmp_path: Path,
|
||||
):
|
||||
module = _patched_transcription_module(tmp_path)
|
||||
prepared, error = _convert(
|
||||
module,
|
||||
tmp_path,
|
||||
_late_speech_upload(preserve_header=True),
|
||||
".webm",
|
||||
)
|
||||
|
||||
assert error is None
|
||||
assert prepared is not None
|
||||
output = Path(prepared)
|
||||
assert output.is_file()
|
||||
assert output.read_bytes().startswith(b"RIFF")
|
||||
|
||||
|
||||
def test_invalid_browser_upload_returns_bounded_error_without_ffmpeg_spam(
|
||||
tmp_path: Path,
|
||||
):
|
||||
module = _patched_transcription_module(tmp_path)
|
||||
prepared, error = _convert(
|
||||
module,
|
||||
tmp_path,
|
||||
_late_speech_upload(preserve_header=False),
|
||||
".webm",
|
||||
)
|
||||
|
||||
assert prepared is None
|
||||
assert error == SAFE_CONVERSION_ERROR
|
||||
assert len(error) < 128
|
||||
lowered = error.lower()
|
||||
assert "ffmpeg version" not in lowered
|
||||
assert "configuration:" not in lowered
|
||||
assert "/tmp/" not in lowered
|
||||
|
||||
|
||||
def test_local_conversion_accepts_browser_fallback_containers(tmp_path: Path):
|
||||
module = _patched_transcription_module(tmp_path)
|
||||
formats = ((".ogg", "libopus"), (".mp4", "aac"))
|
||||
|
||||
for suffix, codec in formats:
|
||||
source = tmp_path / f"source{suffix}"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=0.4",
|
||||
"-c:a",
|
||||
codec,
|
||||
str(source),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
prepared, error = _convert(module, tmp_path, source.read_bytes(), suffix)
|
||||
assert error is None
|
||||
assert prepared is not None
|
||||
assert Path(prepared).read_bytes().startswith(b"RIFF")
|
||||
|
||||
|
||||
def test_browser_uses_actual_recorder_mime_and_supported_extensions():
|
||||
source = VOICE_JS.read_text(encoding="utf-8")
|
||||
|
||||
assert "let initialChunk=null" in source
|
||||
assert "if(initialChunk){chunks.push(initialChunk);initialChunk=null;}" in source
|
||||
assert "let recordedMime=recorder.mimeType||mime||''" in source
|
||||
assert "if(event.data.type) recordedMime=event.data.type" in source
|
||||
assert "audio/mp4;codecs=mp4a.40.2" in source
|
||||
assert "if(normalized.indexOf('ogg')>=0) return 'ogg'" in source
|
||||
assert "if(normalized.indexOf('mp4')>=0) return 'mp4'" in source
|
||||
|
||||
|
||||
def test_stt_patch_is_built_fail_closed_into_the_webui_image(tmp_path: Path):
|
||||
dockerfile = (ROOT / "dockerfiles/Dockerfile.hermes-webui").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "COPY dockerfiles/hermes-webui-stt-patch.py" in dockerfile
|
||||
assert "python /tmp/hermes-webui-stt-patch.py" in dockerfile
|
||||
assert "/opt/hermes/tools/transcription_tools.py" in dockerfile
|
||||
|
||||
drifted = tmp_path / "drifted-agent"
|
||||
shutil.copytree(AGENT_FIXTURE, drifted)
|
||||
transcription = drifted / "tools/transcription_tools.py"
|
||||
source = transcription.read_text(encoding="utf-8")
|
||||
transcription.write_text(
|
||||
source.replace("Failed to convert audio for local STT", "upstream drift", 1),
|
||||
encoding="utf-8",
|
||||
)
|
||||
env = os.environ.copy()
|
||||
env["HERMES_AGENT_PATCH_ROOT"] = str(drifted)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(STT_PATCHER)],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
assert "patch context changed" in result.stderr
|
||||
@ -16,6 +16,9 @@ PATCHER = ROOT / "dockerfiles/hermes-webui-atlas-patch.py"
|
||||
VOICE_JS = ROOT / "dockerfiles/hermes-webui-atlas-voice.js"
|
||||
VOICE_CSS = ROOT / "dockerfiles/hermes-webui-atlas-voice.css"
|
||||
DOM_PROBE = ROOT / "testing/probes/hermes_voice_instrument_probe.js"
|
||||
MEDIARECORDER_FIXTURE = (
|
||||
ROOT / "testing/fixtures/mediarecorder/chromium-webm-opus.json"
|
||||
)
|
||||
|
||||
|
||||
def _patched_fixture(tmp_path: Path) -> Path:
|
||||
@ -86,7 +89,7 @@ def test_visual_states_have_distinct_layers_finite_error_and_reduced_motion():
|
||||
|
||||
def test_dom_probe_exercises_actual_injected_voice_script():
|
||||
result = subprocess.run(
|
||||
["node", str(DOM_PROBE), str(VOICE_JS)],
|
||||
["node", str(DOM_PROBE), str(VOICE_JS), str(MEDIARECORDER_FIXTURE)],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user