From 546e960185669c281f445b7b0fa79c9510fde94e Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 20:16:57 -0300 Subject: [PATCH] hermes(stt-client): retry a brief STT outage, never dump a traceback A transcription that landed while the private Whisper service was restarting (an image roll) crashed hermes_stt_client.py with a raw urllib ConnectionRefused traceback that got dumped into the conversation. The client now retries the request with backoff (up to 5 attempts, ~10s - long enough to ride an STT pod restart) and, on a persistent outage, exits with one concise line ('speech transcription unavailable...') instead of a stack trace. Delivered via the coordinator ConfigMap; the next reconcile picks it up. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- services/hermes/scripts/hermes_stt_client.py | 41 +++++++++++++++++-- .../test_hermes_voice_language_routing.py | 39 ++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/services/hermes/scripts/hermes_stt_client.py b/services/hermes/scripts/hermes_stt_client.py index 96f7081d..2e521f2c 100644 --- a/services/hermes/scripts/hermes_stt_client.py +++ b/services/hermes/scripts/hermes_stt_client.py @@ -8,9 +8,17 @@ import json import mimetypes import os import secrets +import sys +import time from pathlib import Path +from urllib.error import URLError from urllib.request import Request, urlopen +# The private Whisper service briefly restarts on image rolls; a transcription +# request that lands in that window must retry rather than crash the turn. +_MAX_ATTEMPTS = 5 +_RETRY_BACKOFF_SECONDS = (0.5, 1.5, 3.0, 5.0) + LANGUAGE_SUFFIX = ".language" @@ -70,6 +78,29 @@ def _multipart(audio: Path, language: str, model: str) -> tuple[bytes, str]: return b"".join(chunks), boundary +def _transcribe_with_retry(request: Request) -> dict: + """POST to the Whisper service, retrying a brief outage (a service roll). + + A connection error means the private STT service is momentarily down + (e.g. rolling to a new image); retry with backoff instead of surfacing a + Python traceback into the conversation. On exhaustion, raise a plain + RuntimeError so ``main`` can print one short line, not a stack trace. + """ + last_error: Exception | None = None + for attempt in range(_MAX_ATTEMPTS): + try: + with urlopen(request, timeout=120) as response: + return json.loads(response.read().decode("utf-8")) + except (URLError, ConnectionError, TimeoutError, OSError) as error: + last_error = error + if attempt < _MAX_ATTEMPTS - 1: + delay = _RETRY_BACKOFF_SECONDS[min(attempt, len(_RETRY_BACKOFF_SECONDS) - 1)] + time.sleep(delay) + raise RuntimeError( + "the speech service is temporarily unavailable; please try again" + ) from last_error + + def main() -> None: """Transcribe one file and emit the .txt contract Hermes expects.""" parser = argparse.ArgumentParser() @@ -92,8 +123,7 @@ def main() -> None: }, method="POST", ) - with urlopen(request, timeout=120) as response: - result = json.loads(response.read().decode("utf-8")) + result = _transcribe_with_retry(request) transcript = str(result.get("text") or "").strip() _write_result( args.output_dir, @@ -104,4 +134,9 @@ def main() -> None: if __name__ == "__main__": - main() + try: + main() + except RuntimeError as error: + # A concise message, never a traceback, reaches the conversation. + print(f"speech transcription unavailable: {error}", file=sys.stderr) + sys.exit(1) diff --git a/testing/tests/test_hermes_voice_language_routing.py b/testing/tests/test_hermes_voice_language_routing.py index 1a07ce40..6a9373e8 100644 --- a/testing/tests/test_hermes_voice_language_routing.py +++ b/testing/tests/test_hermes_voice_language_routing.py @@ -365,6 +365,45 @@ def test_stt_language_absent_when_whisper_omits_it(monkeypatch): # --------------------------------------------------------------------------- +def test_stt_client_retries_a_brief_outage_then_fails_cleanly(monkeypatch, tmp_path): + """A momentary connection refusal (service roll) retries; a persistent + outage raises a plain RuntimeError, never a urllib traceback.""" + module = _load_stt_client() + monkeypatch.setattr(module.time, "sleep", lambda *_a, **_k: None) + + calls = {"n": 0} + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b'{"text": "recovered", "language": "en"}' + + def flaky(_request, timeout=None): + calls["n"] += 1 + if calls["n"] < 3: + raise ConnectionRefusedError(111, "Connection refused") + return _Resp() + + monkeypatch.setattr(module, "urlopen", flaky) + req = module.Request("http://stt.invalid/", data=b"", method="POST") + assert module._transcribe_with_retry(req)["text"] == "recovered" + assert calls["n"] == 3 + + def always_refused(_request, timeout=None): + raise ConnectionRefusedError(111, "Connection refused") + + monkeypatch.setattr(module, "urlopen", always_refused) + import pytest as _pytest + + with _pytest.raises(RuntimeError, match="temporarily unavailable"): + module._transcribe_with_retry(req) + + def test_stt_client_writes_language_sidecar_beside_the_txt_contract(tmp_path): module = _load_stt_client() module._write_result(tmp_path, "voice-input", "Как дела?", "ru")