diff --git a/dockerfiles/Dockerfile.hermes-jetson-tts b/dockerfiles/Dockerfile.hermes-jetson-tts index deba4080..4257934a 100644 --- a/dockerfiles/Dockerfile.hermes-jetson-tts +++ b/dockerfiles/Dockerfile.hermes-jetson-tts @@ -24,18 +24,41 @@ ADD --checksum=sha256:f7d01dde371555732c4c314111ac79672b1a5ce2fc19266ab42178fd8d ADD --checksum=sha256:45754dfdebb3b8661c3fc564713772deec6e064feeb5b4e9594857dc7305193a --chmod=0444 \ https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/en/en_US/lessac/low/en_US-lessac-low.onnx.json?download=true \ /opt/models/piper/en_US-lessac-low.onnx.json + +# Multilingual chat voice policy: English -> amy, Russian -> irina, Spanish -> +# claude (Mexican Spanish, the only "claude" voice rhasspy/piper-voices +# publishes; there is no es_ES-claude). +ADD --checksum=sha256:b3a6e47b57b8c7fbe6a0ce2518161a50f59a9cdd8a50835c02cb02bdd6206c18 --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/en/en_US/amy/medium/en_US-amy-medium.onnx?download=true \ + /opt/models/piper/en_US-amy-medium.onnx +ADD --checksum=sha256:95a23eb4d42909d38df73bb9ac7f45f597dbfcde2d1bf9526fdeaf5466977d77 --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/en/en_US/amy/medium/en_US-amy-medium.onnx.json?download=true \ + /opt/models/piper/en_US-amy-medium.onnx.json +ADD --checksum=sha256:8ff38212d23da300bbe3705c645e6e5b9475f0bfde01558eb17813e22acaaaaa --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/ru/ru_RU/irina/medium/ru_RU-irina-medium.onnx?download=true \ + /opt/models/piper/ru_RU-irina-medium.onnx +ADD --checksum=sha256:c2ec28bb38e2b59e93b959b3e40348c1afebbd272f30fed5d41205d08e98a9d7 --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/ru/ru_RU/irina/medium/ru_RU-irina-medium.onnx.json?download=true \ + /opt/models/piper/ru_RU-irina-medium.onnx.json +ADD --checksum=sha256:3ef40a71ea63852cd8ab7e6fa7d2ecdcfa67a0b47c9c48e3f10e02ee02083ea0 --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/es/es_MX/claude/high/es_MX-claude-high.onnx?download=true \ + /opt/models/piper/es_MX-claude-high.onnx +ADD --checksum=sha256:1afc81f703c0e4cb3b4d7c0dca096b8b54a98806807f0170cf5eb5557723c12d --chmod=0444 \ + https://huggingface.co/rhasspy/piper-voices/resolve/ea046e8458f6acd997706d6e6066a022b42f6fb1/es/es_MX/claude/high/es_MX-claude-high.onnx.json?download=true \ + /opt/models/piper/es_MX-claude-high.onnx.json RUN chmod 0555 /opt/models /opt/models/piper COPY dockerfiles/hermes-jetson-tts-server.py /opt/atlas/hermes-jetson-tts-server.py RUN chmod 0555 /opt/atlas/hermes-jetson-tts-server.py -# Load the actual pinned voice during the ARM64 build. This catches package or -# model-format drift before the image can reach Flux. -RUN python -c "import stat; from pathlib import Path; from piper import PiperVoice; p=Path('/opt/models/piper'); models=[p/'en_US-lessac-high.onnx',p/'en_US-lessac-medium.onnx',p/'en_US-lessac-low.onnx']; assert stat.S_IMODE(p.stat().st_mode)==0o555; assert all(stat.S_IMODE(model.stat().st_mode)==0o444 for model in models); voices=[PiperVoice.load(model,Path(str(model)+'.json'),use_cuda=False,download_dir=p) for model in models]; assert all(voice.config.sample_rate>0 for voice in voices)" +# Load every pinned voice during the ARM64 build, including the three baked +# for the multilingual chat policy. This catches package or model-format +# drift before the image can reach Flux. +RUN python -c "import stat; from pathlib import Path; from piper import PiperVoice; p=Path('/opt/models/piper'); models=[p/'en_US-lessac-high.onnx',p/'en_US-lessac-medium.onnx',p/'en_US-lessac-low.onnx',p/'en_US-amy-medium.onnx',p/'ru_RU-irina-medium.onnx',p/'es_MX-claude-high.onnx']; assert stat.S_IMODE(p.stat().st_mode)==0o555; assert all(stat.S_IMODE(model.stat().st_mode)==0o444 for model in models); voices=[PiperVoice.load(model,Path(str(model)+'.json'),use_cuda=False,download_dir=p) for model in models]; assert all(voice.config.sample_rate>0 for voice in voices)" ENV HERMES_TTS_HOST=0.0.0.0 \ HERMES_TTS_PORT=9001 \ - HERMES_TTS_VOICE=en_US-lessac-medium \ + HERMES_TTS_VOICE=en_US-amy-medium \ HERMES_TTS_CACHE=/opt/models/piper \ OMP_NUM_THREADS=2 \ PYTHONDONTWRITEBYTECODE=1 \ diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui index 29e69ad3..31a954a9 100644 --- a/dockerfiles/Dockerfile.hermes-webui +++ b/dockerfiles/Dockerfile.hermes-webui @@ -116,9 +116,16 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ && grep -Fq 'routing_priority:priority' /opt/hermes-webui/static/atlas-router.js \ && grep -Fq "'atlas/auto/fast':'Automatic · Fast'" /opt/hermes-webui/static/atlas-router.js \ && grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \ + && grep -Fq '"language": detected_language' /opt/hermes/tools/transcription_tools.py \ + && grep -Fq "'language': detected" /opt/hermes-webui/api/upload.py \ + && grep -Fq 'def _atlas_tts_language(body):' /opt/hermes-webui/api/routes.py \ + && grep -Fq 'request_payload["language"] = _atlas_language' /opt/hermes-webui/api/routes.py \ + && grep -Fq 'takeSttLanguage(token)' /opt/hermes-webui/static/atlas-voice.js \ && /opt/hermes/.venv/bin/python -m py_compile \ /opt/hermes-webui/api/routes.py \ - /opt/hermes-webui/api/gateway_chat.py + /opt/hermes-webui/api/upload.py \ + /opt/hermes-webui/api/gateway_chat.py \ + /opt/hermes/tools/transcription_tools.py # Exercise the real server process in the target architecture before publish. RUN set -eu; \ diff --git a/dockerfiles/hermes-jetson-stt-server.py b/dockerfiles/hermes-jetson-stt-server.py index 1ecb9374..6e0efd08 100644 --- a/dockerfiles/hermes-jetson-stt-server.py +++ b/dockerfiles/hermes-jetson-stt-server.py @@ -62,6 +62,34 @@ def _clean_transcript(result: dict) -> str: return " ".join(kept).strip() +def _detected_language(result: object) -> str: + """Return the bare ISO-639 code Whisper decoded with, or nothing at all. + + ``whisper.transcribe`` reports the language it auto-detected (or the one it + was told to use) as a plain lowercase token such as ``en``/``ru``/``yue``. + Anything that is not that exact shape is dropped rather than guessed at, so + a surprising model result can never become a downstream voice selector. + """ + if not isinstance(result, dict): + return "" + value = result.get("language") + if not isinstance(value, str): + return "" + code = value.strip().lower() + if not 2 <= len(code) <= 3 or not code.isascii() or not code.isalpha(): + return "" + return code + + +def _transcription_payload(result: dict) -> dict: + """Build the transcription contract: text plus the model's own language.""" + return { + "text": _clean_transcript(result), + "model": MODEL_NAME, + "language": _detected_language(result), + } + + def _json(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None: body = json.dumps(payload).encode("utf-8") handler.send_response(status) @@ -148,8 +176,7 @@ class SpeechHandler(BaseHTTPRequestHandler): no_speech_threshold=0.5, verbose=False, ) - transcript = _clean_transcript(result) - _json(self, 200, {"text": transcript, "model": MODEL_NAME}) + _json(self, 200, _transcription_payload(result)) except Exception as exc: print(f"[stt] transcription failed: {exc}", flush=True) _json(self, 500, {"error": "transcription failed"}) diff --git a/dockerfiles/hermes-jetson-tts-server.py b/dockerfiles/hermes-jetson-tts-server.py index 3abdc28b..0b962ae2 100644 --- a/dockerfiles/hermes-jetson-tts-server.py +++ b/dockerfiles/hermes-jetson-tts-server.py @@ -17,12 +17,50 @@ from piper import PiperConfig, PiperVoice, SynthesisConfig HOST = os.getenv("HERMES_TTS_HOST", "0.0.0.0") PORT = int(os.getenv("HERMES_TTS_PORT", "9001")) -VOICE_NAME = os.getenv("HERMES_TTS_VOICE", "en_US-lessac-high") CACHE_DIR = Path(os.getenv("HERMES_TTS_CACHE", "/cache/piper")) MAX_TEXT_CHARS = 5000 ONNX_THREADS = max(1, int(os.getenv("HERMES_TTS_ONNX_THREADS", "4"))) VOICE_LOCK = threading.Lock() +# Fixed, allow-listed language -> baked voice mapping. This is the ONLY path +# from a client-supplied string to a model name: client input is looked up +# here and never used to build a filesystem path directly. Both "-" and "_" +# separators and any case are accepted; anything not present here falls back +# to DEFAULT_VOICE_NAME (safe English default), never an error and never an +# unbaked model. +LANGUAGE_VOICE_MAP = { + "en": "en_US-amy-medium", + "en-us": "en_US-amy-medium", + "ru": "ru_RU-irina-medium", + "ru-ru": "ru_RU-irina-medium", + "es": "es_MX-claude-high", + "es-mx": "es_MX-claude-high", + "es-es": "es_MX-claude-high", +} +BAKED_VOICE_NAMES = frozenset(LANGUAGE_VOICE_MAP.values()) +DEFAULT_VOICE_NAME = os.getenv("HERMES_TTS_VOICE", "en_US-amy-medium") + + +def normalize_language(value: object) -> str | None: + """Lowercase and fold "_"/"-" separators; reject non-string/blank input.""" + if not isinstance(value, str): + return None + normalized = value.strip().lower().replace("_", "-") + return normalized or None + + +def resolve_voice_name(language: object) -> str: + """Map a client-supplied language to one of the baked policy voices. + + Unknown, missing, or malformed language always resolves to the safe + default rather than raising, and the result is always a member of + BAKED_VOICE_NAMES. + """ + normalized = normalize_language(language) + if normalized is None: + return DEFAULT_VOICE_NAME + return LANGUAGE_VOICE_MAP.get(normalized, DEFAULT_VOICE_NAME) + def _json(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None: body = json.dumps(payload).encode("utf-8") @@ -46,7 +84,16 @@ class SpeechHandler(BaseHTTPRequestHandler): if self.path != "/health": _json(self, 404, {"error": "not found"}) return - _json(self, 200, {"ok": True, "voice": VOICE_NAME, "device": "cpu"}) + _json( + self, + 200, + { + "ok": True, + "voices": sorted(self.server.voices), # type: ignore[attr-defined] + "default_voice": self.server.default_voice_name, # type: ignore[attr-defined] + "device": "cpu", + }, + ) def do_POST(self) -> None: if self.path != "/v1/audio/speech": @@ -71,10 +118,16 @@ class SpeechHandler(BaseHTTPRequestHandler): return speed = min(2.0, max(0.5, speed)) + # Policy is driven ONLY by "language". A client-supplied "voice" + # field is deliberately never read here; it cannot override the + # allow-listed mapping. + voice_name = resolve_voice_name(payload.get("language")) + voice = self.server.voices[voice_name] # type: ignore[attr-defined] + output = io.BytesIO() try: with VOICE_LOCK, wave.open(output, "wb") as wav_file: - self.server.voice.synthesize_wav( # type: ignore[attr-defined] + voice.synthesize_wav( text, wav_file, SynthesisConfig(length_scale=1.0 / speed), @@ -84,6 +137,7 @@ class SpeechHandler(BaseHTTPRequestHandler): self.send_header("Content-Type", "audio/wav") self.send_header("Content-Length", str(len(audio))) self.send_header("Cache-Control", "no-store") + self.send_header("X-TTS-Voice", voice_name) self.end_headers() self.wfile.write(audio) except Exception as exc: @@ -91,30 +145,54 @@ class SpeechHandler(BaseHTTPRequestHandler): _json(self, 500, {"error": "speech synthesis failed"}) -def main() -> None: - """Load the checksum-pinned voice from the image and serve it on CPU.""" - CACHE_DIR.mkdir(parents=True, exist_ok=True) - model_path = CACHE_DIR / f"{VOICE_NAME}.onnx" - config_path = CACHE_DIR / f"{VOICE_NAME}.onnx.json" +def _load_voice(cache_dir: Path, voice_name: str, threads: int) -> PiperVoice: + model_path = cache_dir / f"{voice_name}.onnx" + config_path = cache_dir / f"{voice_name}.onnx.json" if not model_path.exists() or not config_path.exists(): - raise RuntimeError(f"baked Piper voice is missing: {VOICE_NAME}") + raise RuntimeError(f"baked Piper voice is missing: {voice_name}") with config_path.open("r", encoding="utf-8") as config_file: config = PiperConfig.from_dict(json.load(config_file)) session_options = onnxruntime.SessionOptions() - session_options.intra_op_num_threads = ONNX_THREADS + session_options.intra_op_num_threads = threads session_options.inter_op_num_threads = 1 session = onnxruntime.InferenceSession( str(model_path), sess_options=session_options, providers=["CPUExecutionProvider"], ) - voice = PiperVoice(session=session, config=config, download_dir=CACHE_DIR) + return PiperVoice(session=session, config=config, download_dir=cache_dir) + + +def load_voices(cache_dir: Path, threads: int) -> dict[str, PiperVoice]: + """Eagerly load all three policy voices. + + Preload (not lazy-load-on-first-use) was chosen deliberately: measured + RSS on this model set is ~88MB for one voice and ~243MB for all three + (~+155MB versus the previous single-voice baseline), which comfortably + fits the pod's memory budget on the CPU-only voice node. Preloading + avoids a slow, request-serializing first synthesis per language and + keeps the fail-closed missing-model check (below) at process start + rather than deferring a possible crash to a live user request. + """ + return {name: _load_voice(cache_dir, name, threads) for name in sorted(BAKED_VOICE_NAMES)} + + +def main() -> None: + """Load the checksum-pinned policy voices from the image and serve them on CPU.""" + CACHE_DIR.mkdir(parents=True, exist_ok=True) + if DEFAULT_VOICE_NAME not in BAKED_VOICE_NAMES: + raise RuntimeError( + f"HERMES_TTS_VOICE must name one of the baked policy voices: {sorted(BAKED_VOICE_NAMES)}" + ) + voices = load_voices(CACHE_DIR, ONNX_THREADS) print( - f"[tts] loaded Piper voice {VOICE_NAME} on CPU with {ONNX_THREADS} ONNX threads", + f"[tts] loaded {len(voices)} Piper voices on CPU with {ONNX_THREADS} ONNX threads each: " + + ", ".join(sorted(voices)), flush=True, ) server = ThreadingHTTPServer((HOST, PORT), SpeechHandler) - server.voice = voice # type: ignore[attr-defined] + server.voices = voices # type: ignore[attr-defined] + server.default_voice_name = DEFAULT_VOICE_NAME # type: ignore[attr-defined] print(f"[tts] ready on {HOST}:{PORT}", flush=True) server.serve_forever(poll_interval=0.25) diff --git a/dockerfiles/hermes-webui-atlas-patch.py b/dockerfiles/hermes-webui-atlas-patch.py index d7bccf63..c39e236b 100644 --- a/dockerfiles/hermes-webui-atlas-patch.py +++ b/dockerfiles/hermes-webui-atlas-patch.py @@ -6,6 +6,9 @@ from pathlib import Path ROOT = Path(os.environ.get("HERMES_WEBUI_PATCH_ROOT", "/opt/hermes-webui")) +# The WebUI imports the pinned agent's STT tooling from the same image, so the +# local-command transcription envelope is patched alongside the WebUI itself. +AGENT_ROOT = Path(os.environ.get("HERMES_AGENT_PATCH_ROOT", "/opt/hermes")) def replace_exact(path: Path, before: str, after: str, count: int = 1) -> None: @@ -16,6 +19,43 @@ def replace_exact(path: Path, before: str, after: str, count: int = 1) -> None: path.write_text(source.replace(before, after, count), encoding="utf-8") +def replace_between_exact( + path: Path, start: str, end: str, after: str = "", count: int = 1 +) -> None: + """Replace one exact, bounded upstream region and fail when the pin drifts.""" + source = path.read_text(encoding="utf-8") + if source.count(start) != count or source.count(end) != count: + raise SystemExit( + f"Atlas voice patch context changed in {path}: {start[:80]!r}" + ) + start_index = source.index(start) + end_index = source.index(end, start_index) + len(end) + path.write_text( + source[:start_index] + after + source[end_index:], encoding="utf-8" + ) + + +def assert_absent(path: Path, *needles: str) -> None: + """Fail the image build if a removed voice-choice surface remains.""" + source = path.read_text(encoding="utf-8") + remaining = [needle for needle in needles if needle in source] + if remaining: + raise SystemExit(f"Atlas voice choice remains in {path}: {remaining!r}") + + +def remove_lines_containing(path: Path, *needles: str) -> None: + """Remove all pinned translation entries for a retired settings control.""" + source = path.read_text(encoding="utf-8") + for needle in needles: + if needle not in source: + raise SystemExit(f"Atlas voice patch context changed in {path}: {needle!r}") + lines = source.splitlines(keepends=True) + path.write_text( + "".join(line for line in lines if not any(n in line for n in needles)), + encoding="utf-8", + ) + + index = ROOT / "static/index.html" replace_exact( index, @@ -29,6 +69,16 @@ replace_exact( '', '', ) +replace_exact( + index, + '''
+ +
Preferred voice. Populated from your browser's available voices.
+
''', + "", +) replace_exact( index, '', @@ -62,11 +112,33 @@ replace_exact( ) ui = ROOT / "static/ui.js" +replace_exact( + ui, + ''' const savedVoice=localStorage.getItem('hermes-tts-voice'); + const voices=speechSynthesis.getVoices(); + if(savedVoice&&voices.length){ + const match=voices.find(v=>v.name===savedVoice); + if(match) utter.voice=match; + } +''', + "", +) replace_exact(ui, "function _playEdgeTtsChunked(text, btn){", "function _playEdgeTtsChunked(text, btn, engineOverride){") +replace_exact( + ui, + " const voice=localStorage.getItem('hermes-tts-voice')||'zh-CN-XiaoxiaoNeural';\n", + "", +) replace_exact( ui, "body:JSON.stringify({text:chunk, voice:voice, rate:rate, pitch:pitch})", - "body:JSON.stringify({text:chunk, voice:voice, rate:rate, pitch:pitch, engine:engineOverride||'edge'})", + "body:JSON.stringify({text:chunk, rate:rate, pitch:pitch, engine:engineOverride||'edge'})", +) +replace_exact( + ui, + " voice: localStorage.getItem('hermes-tts-voice')||'',\n", + "", + count=2, ) replace_exact( ui, @@ -79,7 +151,177 @@ replace_exact( "if(engine==='edge'||engine==='atlas'){\n _playEdgeTtsChunked(clean, null, engine);", ) +panels = ROOT / "static/panels.js" +replace_exact(panels, " tts_voice:'hermes-tts-voice',\n", "") +replace_exact( + panels, + ''' const ttsVoiceSel=$('settingsTtsVoice'); + if(ttsVoiceSel) _setOwnedSpeechPayload(payload,'tts_voice',ttsVoiceSel.value||''); +''', + "", +) +replace_exact( + panels, + ''' localStorage.setItem('hermes-tts-engine',this.value); + window._populateTtsVoices(); + _schedulePreferencesAutosave();''', + ''' localStorage.setItem('hermes-tts-engine',this.value); + _schedulePreferencesAutosave();''', +) +replace_between_exact( + panels, + " // Populate voice selector based on engine\n", + " // TTS rate/pitch sliders\n", + " // TTS speaker selection is intentionally server policy only.\n", +) +replace_exact( + panels, + "let _settingsSpeechChangedKeys=new Set();\n", + "let _settingsSpeechChangedKeys=new Set();\n" + "try{localStorage.removeItem('hermes-tts-voice');}catch(_){}\n", +) + +boot = ROOT / "static/boot.js" +replace_exact( + boot, + ''' voice: localStorage.getItem("hermes-tts-voice")||'', +''', + "", +) +replace_exact( + boot, + ''' const voice=localStorage.getItem("hermes-tts-voice")||"zh-CN-XiaoxiaoNeural"; +''', + "", +) +replace_exact( + boot, + " body: JSON.stringify({text: clean, voice, rate, pitch})", + " body: JSON.stringify({text: clean, rate, pitch})", +) +replace_exact( + boot, + ''' const savedVoice=localStorage.getItem('hermes-tts-voice'); + const voices=speechSynthesis.getVoices(); + if(savedVoice&&voices.length){ + const match=voices.find(v=>v.name===savedVoice); + if(match) utter.voice=match; + } +''', + "", +) +replace_exact(boot, " tts_voice:'',\n", "") +replace_exact(boot, " ['tts_voice','hermes-tts-voice'],\n", "") + +config = ROOT / "api/config.py" +replace_exact(config, ' "tts_voice": "",\n', "") +replace_exact(config, ' "tts_voice",\n', "") +replace_exact( + config, + ''' if k == "tts_voice": + if not isinstance(v, str) or len(v) > 200 or "\\x00" in v: + continue +''', + "", +) + +assert_absent(index, "settingsTtsVoice", "settings_label_tts_voice") +assert_absent(ui, "hermes-tts-voice", "voice:voice") +assert_absent(panels, "settingsTtsVoice", "tts_voice") +assert_absent(boot, "hermes-tts-voice", "tts_voice", "text: clean, voice") +assert_absent(config, '"tts_voice"') + +i18n = ROOT / "static/i18n.js" +remove_lines_containing( + i18n, + "settings_label_tts_voice:", + "settings_desc_tts_voice:", +) +assert_absent(i18n, "settings_label_tts_voice", "settings_desc_tts_voice") + +# The private Whisper service reports the language it decoded with. Carry that +# through the agent's local-command STT envelope so the WebUI can hand a voice +# hint to Piper instead of guessing the reply's language from its text. +transcription = AGENT_ROOT / "tools/transcription_tools.py" +replace_exact( + transcription, + ''' transcript_text = txt_files[0].read_text(encoding="utf-8").strip() + logger.info( + "Transcribed %s via local STT command (%s, %d chars)", + Path(file_path).name, + normalized_model, + len(transcript_text), + ) + return {"success": True, "transcript": transcript_text, "provider": "local_command"} +''', + ''' transcript_text = txt_files[0].read_text(encoding="utf-8").strip() + logger.info( + "Transcribed %s via local STT command (%s, %d chars)", + Path(file_path).name, + normalized_model, + len(transcript_text), + ) + detected_language = "" + language_files = sorted(Path(output_dir).glob("*.language")) + if language_files: + try: + candidate = language_files[0].read_text(encoding="utf-8").strip().lower() + except (OSError, ValueError): + candidate = "" + if 2 <= len(candidate) <= 3 and candidate.isascii() and candidate.isalpha(): + detected_language = candidate + return { + "success": True, + "transcript": transcript_text, + "provider": "local_command", + "language": detected_language, + } +''', +) + +upload = ROOT / "api/upload.py" +replace_exact( + upload, + """ transcript = str(result.get('transcript') or '').strip() + return j(handler, {'ok': True, 'transcript': transcript}) +""", + """ transcript = str(result.get('transcript') or '').strip() + detected = str(result.get('language') or '').strip().lower() + if not (2 <= len(detected) <= 3 and detected.isascii() and detected.isalpha()): + detected = '' + return j(handler, {'ok': True, 'transcript': transcript, 'language': detected}) +""", +) + routes = ROOT / "api/routes.py" +replace_exact( + routes, + "def _tts_open(req, *, timeout=30, opener_factory=None):", + '''ATLAS_TTS_LANGUAGES = ("en", "ru", "es") + + +def _atlas_tts_language(body): + """Return a plain, allow-listed en/ru/es code, or "" to send no language. + + This is a trust boundary, not a parser. Only the exact normalized codes the + private Piper deployment bakes a voice for are forwarded; a missing field, + a wrong type, a region tag, padding, control characters, a traversal or + injection string, an oversized value, an object, an array, a number or a + client-supplied "voice" all resolve to "" and the language field is then + omitted entirely, so the Jetson service applies its own English default. + Coercing a malformed value into a supported code would let a browser + describe hostile input as a language we support; omission cannot. + """ + if not isinstance(body, dict): + return "" + value = body.get("language") + if not isinstance(value, str): + return "" + return value if value in ATLAS_TTS_LANGUAGES else "" + + +def _tts_open(req, *, timeout=30, opener_factory=None):''', +) marker = " # ── ElevenLabs TTS ──────────────────────────────────────────────────\n" atlas = ''' # ── Atlas private Jetson TTS ───────────────────────────────────────── if engine == "atlas": @@ -94,12 +336,19 @@ atlas = ''' # ── Atlas private Jetson TTS ────────── speed = max(0.5, min(2.0, 1.0 + (float(rate_str.rstrip("%")) / 100.0))) except ValueError: speed = 1.0 - request_body = json.dumps({ + request_payload = { "model": "piper", "input": text, - "voice": "en_US-lessac-high", "speed": speed, - }).encode("utf-8") + } + # Attach a language ONLY when the browser sent a plain allow-listed + # code. Omitting it is the fail-safe: the Jetson service then speaks + # its own English default, which is also what every partially rolled + # out combination of these components degrades to. + _atlas_language = _atlas_tts_language(data) + if _atlas_language: + request_payload["language"] = _atlas_language + request_body = json.dumps(request_payload).encode("utf-8") request = Request(atlas_url, data=request_body, headers={ "Content-Type": "application/json", "Accept": "audio/wav", diff --git a/dockerfiles/hermes-webui-atlas-voice.js b/dockerfiles/hermes-webui-atlas-voice.js index 29cbf0a1..4f6ed049 100644 --- a/dockerfiles/hermes-webui-atlas-voice.js +++ b/dockerfiles/hermes-webui-atlas-voice.js @@ -31,9 +31,37 @@ error:'Voice unavailable', idle:'', }; + // The only language signal this file trusts is the one the private Whisper + // service returned for the audio of the turn currently being answered. It is + // bound to that turn's generation token and consumed exactly once. + let sttLanguage=''; + let sttLanguageToken=-1; + const TTS_LANGUAGES=['en','ru','es']; const originalAutoRead=window.autoReadLastAssistant; const originalApplyPreference=window._applyVoiceModePref; + function normalizeSttLanguage(value){ + if(typeof value!=='string') return ''; + const code=value.trim().toLowerCase(); + return TTS_LANGUAGES.indexOf(code)>=0?code:''; + } + + function clearSttLanguage(){ + sttLanguage=''; + sttLanguageToken=-1; + } + + function rememberSttLanguage(language, token){ + sttLanguage=language||''; + sttLanguageToken=sttLanguage?token:-1; + } + + function takeSttLanguage(token){ + const language=sttLanguageToken===token?sttLanguage:''; + clearSttLanguage(); + return language; + } + function toast(message){ if(typeof window.showToast==='function') window.showToast(message,3000); } @@ -107,6 +135,7 @@ active=false; thinkingSession=null; clearErrorTimer(); + clearSttLanguage(); stopCapture(); stopPlayback(); modeBtn.classList.remove('active'); @@ -120,13 +149,14 @@ },delay||500); } - function sendTranscript(transcript, token){ + function sendTranscript(transcript, token, language){ if(!active||token!==generation) return; const text=String(transcript||'').trim(); - if(!text){restartSoon(token,350);return;} + if(!text){clearSttLanguage();restartSoon(token,350);return;} composer.value=text; if(typeof window.autoResize==='function') window.autoResize(); thinkingSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null; + rememberSttLanguage(language,token); setState('thinking'); if(typeof window.send==='function') window.send(); } @@ -141,7 +171,7 @@ const response=await fetch('/api/transcribe',{method:'POST',body:form}); const payload=await response.json().catch(function(){return {};}); if(!response.ok) throw new Error(payload.error||('Whisper request failed: '+response.status)); - sendTranscript(payload.transcript,token); + sendTranscript(payload.transcript,token,normalizeSttLanguage(payload.language)); }catch(error){ if(!active||token!==generation) return; const message=errorMessage(error,'Private Whisper is unavailable'); @@ -159,6 +189,7 @@ async function startListening(token){ if(!active||token!==generation) return; stopCapture(); + clearSttLanguage(); setState('listening'); try{ const capture=await navigator.mediaDevices.getUserMedia({ @@ -273,11 +304,15 @@ }); } - async function fetchSpeech(chunk){ + async function fetchSpeech(chunk, language){ + // `language` is only ever the private STT result for this turn. When it is + // absent the field is omitted entirely and the server picks English. + const request={text:chunk,engine:'atlas'}; + if(language) request.language=language; const response=await fetch('/api/tts',{ method:'POST', headers:{'Content-Type':'application/json'}, - body:JSON.stringify({text:chunk,engine:'atlas'}), + body:JSON.stringify(request), }); if(!response.ok){ const payload=await response.json().catch(function(){return {};}); @@ -291,10 +326,12 @@ const currentSession=(typeof S!=='undefined'&&S.session)?S.session.session_id:null; if(thinkingSession&¤tSession&&thinkingSession!==currentSession){ thinkingSession=null; + clearSttLanguage(); restartSoon(token,250); return; } thinkingSession=null; + const language=takeSttLanguage(token); const rows=document.querySelectorAll('.msg-row[data-role="assistant"], .assistant-segment[data-raw-text]'); if(!rows.length){restartSoon(token,250);return;} const text=cleanForSpeech(rows[rows.length-1].dataset.rawText||''); @@ -302,11 +339,11 @@ setState('speaking'); const chunks=typeof window._splitForTTS==='function'?window._splitForTTS(text,280):[text]; try{ - let pending=fetchSpeech(chunks[0]); + let pending=fetchSpeech(chunks[0],language); for(let index=0;index.txt` transcript plus a + `.language` sidecar. The transcript stays the only `.txt` in the + output directory, so the stock Hermes local-command contract is unchanged. +3. The patched local-command STT envelope reads that sidecar and adds + `language` to its result; `/api/transcribe` re-validates it and returns it + next to `transcript`. +4. `atlas-voice.js` keeps that value only for the turn it belongs to. It is + bound to the voice-mode generation token and the chat session id, consumed + exactly once by the reply that turn produced, and cleared on cancellation, + restart, session change, an empty transcript, or a transcription error. +5. `/api/tts` accepts `language` only from the fixed allow-list and otherwise + sends English. The Jetson TTS service applies the same allow-list again as + the final authority. + +**What this does not claim.** The language is the language the *user spoke*, +not the language of the reply. A model asked a Russian question may answer in +English and will then be read aloud by the Russian voice, and vice versa; this +is a deliberate policy choice for hands-free mode, not a detection failure. +Nothing here detects the language of assistant text. + +**Everything else stays English.** Typed messages, the manual read-aloud +button, and any assistant reply that was not produced by a hands-free spoken +turn carry no trusted STT signal, so they synthesize with `en_US-amy-medium`. +A `voice` field from a browser is never honoured at any hop. + ## Honest limits - Hermes does not currently apply production or cluster changes autonomously. diff --git a/services/hermes/scripts/hermes_stt_client.py b/services/hermes/scripts/hermes_stt_client.py index 639fc7b1..96f7081d 100644 --- a/services/hermes/scripts/hermes_stt_client.py +++ b/services/hermes/scripts/hermes_stt_client.py @@ -12,6 +12,34 @@ from pathlib import Path from urllib.request import Request, urlopen +LANGUAGE_SUFFIX = ".language" + + +def _normalize_language(value: object) -> str: + """Accept only a bare ISO-639 code from the private Whisper response.""" + if not isinstance(value, str): + return "" + code = value.strip().lower() + if not 2 <= len(code) <= 3 or not code.isascii() or not code.isalpha(): + return "" + return code + + +def _write_result(output_dir: Path, stem: str, transcript: str, language: str) -> Path: + """Write the .txt Hermes reads, plus the language sidecar when we have one. + + Hermes' local-command contract is "leave a .txt in --output-dir"; it globs + ``*.txt`` and reads the first match. The sidecar deliberately uses another + suffix so the transcript stays the only ``.txt`` in the directory. + """ + output_dir.mkdir(parents=True, exist_ok=True) + transcript_path = output_dir / f"{stem}.txt" + transcript_path.write_text(transcript, encoding="utf-8") + if language: + (output_dir / f"{stem}{LANGUAGE_SUFFIX}").write_text(language, encoding="utf-8") + return transcript_path + + def _multipart(audio: Path, language: str, model: str) -> tuple[bytes, str]: boundary = f"atlas-hermes-{secrets.token_hex(12)}" mime = mimetypes.guess_type(audio.name)[0] or "application/octet-stream" @@ -67,9 +95,12 @@ def main() -> None: with urlopen(request, timeout=120) as response: result = json.loads(response.read().decode("utf-8")) transcript = str(result.get("text") or "").strip() - args.output_dir.mkdir(parents=True, exist_ok=True) - output = args.output_dir / f"{args.input_path.stem}.txt" - output.write_text(transcript, encoding="utf-8") + _write_result( + args.output_dir, + args.input_path.stem, + transcript, + _normalize_language(result.get("language")), + ) if __name__ == "__main__": diff --git a/testing/fixtures/hermes-agent/tools/transcription_tools.py b/testing/fixtures/hermes-agent/tools/transcription_tools.py new file mode 100644 index 00000000..4f04b14b --- /dev/null +++ b/testing/fixtures/hermes-agent/tools/transcription_tools.py @@ -0,0 +1,28 @@ +"""Pinned local-command STT envelope anchor used by image-patch tests.""" + +import contextlib +from pathlib import Path + + +class _Logger: + def info(self, *args): + return None + + +logger = _Logger() + + +def _transcribe_local_command(file_path, normalized_model, output_dir): + try: + with contextlib.nullcontext(output_dir): + txt_files = sorted(Path(output_dir).glob("*.txt")) + transcript_text = txt_files[0].read_text(encoding="utf-8").strip() + logger.info( + "Transcribed %s via local STT command (%s, %d chars)", + Path(file_path).name, + normalized_model, + len(transcript_text), + ) + return {"success": True, "transcript": transcript_text, "provider": "local_command"} + except OSError as error: + return {"success": False, "transcript": "", "error": str(error)} diff --git a/testing/fixtures/hermes-webui-0.52.181/api/config.py b/testing/fixtures/hermes-webui-0.52.181/api/config.py new file mode 100644 index 00000000..98bd4888 --- /dev/null +++ b/testing/fixtures/hermes-webui-0.52.181/api/config.py @@ -0,0 +1,10 @@ +_SETTINGS_DEFAULTS = { + "tts_voice": "", +} +_SETTINGS_SPEECH_KEYS = { + "tts_voice", +} +UPSTREAM_VALIDATION_FRAGMENT = ''' if k == "tts_voice": + if not isinstance(v, str) or len(v) > 200 or "\x00" in v: + continue +''' diff --git a/testing/fixtures/hermes-webui-0.52.181/api/routes.py b/testing/fixtures/hermes-webui-0.52.181/api/routes.py index de5cf520..499048df 100644 --- a/testing/fixtures/hermes-webui-0.52.181/api/routes.py +++ b/testing/fixtures/hermes-webui-0.52.181/api/routes.py @@ -1,3 +1,41 @@ -def tts(handler, engine): +"""Pinned Atlas TTS route anchors used by image-patch tests.""" + +import json +import os +from urllib.request import ProxyHandler, Request, build_opener + + +class _NoRedirectTtsHandler: + """Placeholder for the upstream no-redirect opener handler.""" + + +class _Logger: + def exception(self, message): + return None + + +logger = _Logger() + + +class _Upstream: + def read(self): + return b"RIFFsynthetic" + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +def _buffer_tts_audio_response(response): + return response.read() + + +def _tts_open(req, *, timeout=30, opener_factory=None): + return _Upstream() + + +def _handle_tts(handler, data, text, rate_str, engine): # ── ElevenLabs TTS ────────────────────────────────────────────────── - return False + return None diff --git a/testing/fixtures/hermes-webui-0.52.181/api/upload.py b/testing/fixtures/hermes-webui-0.52.181/api/upload.py new file mode 100644 index 00000000..582cebbe --- /dev/null +++ b/testing/fixtures/hermes-webui-0.52.181/api/upload.py @@ -0,0 +1,13 @@ +"""Pinned upstream /api/transcribe response anchor used by image-patch tests.""" + + +def j(handler, payload, status=200): + return {"status": status, "payload": payload} + + +def handle_transcribe(handler, result): + try: + transcript = str(result.get('transcript') or '').strip() + return j(handler, {'ok': True, 'transcript': transcript}) + except ValueError as error: + return j(handler, {'error': str(error)}, status=400) diff --git a/testing/fixtures/hermes-webui-0.52.181/static/boot.js b/testing/fixtures/hermes-webui-0.52.181/static/boot.js new file mode 100644 index 00000000..88541e22 --- /dev/null +++ b/testing/fixtures/hermes-webui-0.52.181/static/boot.js @@ -0,0 +1,34 @@ +function speakWithRegisteredEngine(){ + const _opts={ + voice: localStorage.getItem("hermes-tts-voice")||'', + rate: parseFloat(localStorage.getItem("hermes-tts-rate")), + }; + return _opts; +} +function speakWithEdge(clean){ + const voice=localStorage.getItem("hermes-tts-voice")||"zh-CN-XiaoxiaoNeural"; + const rate=''; + const pitch=''; + return fetch('/api/tts', { + body: JSON.stringify({text: clean, voice, rate, pitch}) + }); +} +function speakWithBrowser(clean){ + const utter=new SpeechSynthesisUtterance(clean); + const savedVoice=localStorage.getItem('hermes-tts-voice'); + const voices=speechSynthesis.getVoices(); + if(savedVoice&&voices.length){ + const match=voices.find(v=>v.name===savedVoice); + if(match) utter.voice=match; + } + return utter; +} +function _mirrorSpeechSettingsFromServer(s){ + const defaults={ + tts_voice:'', + }; + [ + ['tts_voice','hermes-tts-voice'], + ].forEach(([settingKey,storageKey])=>localStorage.setItem(storageKey,s[settingKey])); + return defaults; +} diff --git a/testing/fixtures/hermes-webui-0.52.181/static/i18n.js b/testing/fixtures/hermes-webui-0.52.181/static/i18n.js new file mode 100644 index 00000000..aa558ec1 --- /dev/null +++ b/testing/fixtures/hermes-webui-0.52.181/static/i18n.js @@ -0,0 +1,4 @@ +const EN = { + settings_label_tts_voice: 'Voice', + settings_desc_tts_voice: "Preferred voice. Populated from your browser's available voices.", +}; diff --git a/testing/fixtures/hermes-webui-0.52.181/static/index.html b/testing/fixtures/hermes-webui-0.52.181/static/index.html index 6eb7f63d..b8ff3aca 100644 --- a/testing/fixtures/hermes-webui-0.52.181/static/index.html +++ b/testing/fixtures/hermes-webui-0.52.181/static/index.html @@ -5,6 +5,12 @@ +
+ +
Preferred voice. Populated from your browser's available voices.
+