hermes(stt-client): retry a brief STT outage, never dump a traceback

A transcription that landed while the private Whisper service was
restarting (an image roll) crashed hermes_stt_client.py with a raw
urllib ConnectionRefused traceback that got dumped into the
conversation. The client now retries the request with backoff (up to 5
attempts, ~10s - long enough to ride an STT pod restart) and, on a
persistent outage, exits with one concise line ('speech transcription
unavailable...') instead of a stack trace. Delivered via the coordinator
ConfigMap; the next reconcile picks it up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf

services/hermes/scripts/hermes_stt_client.py

@@ -8,9 +8,17 @@ import json
 import mimetypes
 import os
 import secrets
+import sys
+import time
 from pathlib import Path
+from urllib.error import URLError
 from urllib.request import Request, urlopen
 
+# The private Whisper service briefly restarts on image rolls; a transcription
+# request that lands in that window must retry rather than crash the turn.
+_MAX_ATTEMPTS = 5
+_RETRY_BACKOFF_SECONDS = (0.5, 1.5, 3.0, 5.0)
+
 
 LANGUAGE_SUFFIX = ".language"
 
@@ -70,6 +78,29 @@ def _multipart(audio: Path, language: str, model: str) -> tuple[bytes, str]:
     return b"".join(chunks), boundary
 
 
+def _transcribe_with_retry(request: Request) -> dict:
+    """POST to the Whisper service, retrying a brief outage (a service roll).
+
+    A connection error means the private STT service is momentarily down
+    (e.g. rolling to a new image); retry with backoff instead of surfacing a
+    Python traceback into the conversation. On exhaustion, raise a plain
+    RuntimeError so ``main`` can print one short line, not a stack trace.
+    """
+    last_error: Exception | None = None
+    for attempt in range(_MAX_ATTEMPTS):
+        try:
+            with urlopen(request, timeout=120) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except (URLError, ConnectionError, TimeoutError, OSError) as error:
+            last_error = error
+            if attempt < _MAX_ATTEMPTS - 1:
+                delay = _RETRY_BACKOFF_SECONDS[min(attempt, len(_RETRY_BACKOFF_SECONDS) - 1)]
+                time.sleep(delay)
+    raise RuntimeError(
+        "the speech service is temporarily unavailable; please try again"
+    ) from last_error
+
+
 def main() -> None:
     """Transcribe one file and emit the .txt contract Hermes expects."""
     parser = argparse.ArgumentParser()
@@ -92,8 +123,7 @@ def main() -> None:
         },
         method="POST",
     )
-    with urlopen(request, timeout=120) as response:
-        result = json.loads(response.read().decode("utf-8"))
+    result = _transcribe_with_retry(request)
     transcript = str(result.get("text") or "").strip()
     _write_result(
         args.output_dir,
@@ -104,4 +134,9 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except RuntimeError as error:
+        # A concise message, never a traceback, reaches the conversation.
+        print(f"speech transcription unavailable: {error}", file=sys.stderr)
+        sys.exit(1)

testing/tests/test_hermes_voice_language_routing.py

@@ -365,6 +365,45 @@ def test_stt_language_absent_when_whisper_omits_it(monkeypatch):
 # ---------------------------------------------------------------------------
 
 
+def test_stt_client_retries_a_brief_outage_then_fails_cleanly(monkeypatch, tmp_path):
+    """A momentary connection refusal (service roll) retries; a persistent
+    outage raises a plain RuntimeError, never a urllib traceback."""
+    module = _load_stt_client()
+    monkeypatch.setattr(module.time, "sleep", lambda *_a, **_k: None)
+
+    calls = {"n": 0}
+
+    class _Resp:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *a):
+            return False
+
+        def read(self):
+            return b'{"text": "recovered", "language": "en"}'
+
+    def flaky(_request, timeout=None):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise ConnectionRefusedError(111, "Connection refused")
+        return _Resp()
+
+    monkeypatch.setattr(module, "urlopen", flaky)
+    req = module.Request("http://stt.invalid/", data=b"", method="POST")
+    assert module._transcribe_with_retry(req)["text"] == "recovered"
+    assert calls["n"] == 3
+
+    def always_refused(_request, timeout=None):
+        raise ConnectionRefusedError(111, "Connection refused")
+
+    monkeypatch.setattr(module, "urlopen", always_refused)
+    import pytest as _pytest
+
+    with _pytest.raises(RuntimeError, match="temporarily unavailable"):
+        module._transcribe_with_retry(req)
+
+
 def test_stt_client_writes_language_sidecar_beside_the_txt_contract(tmp_path):
     module = _load_stt_client()
     module._write_result(tmp_path, "voice-input", "Как дела?", "ru")