hermes(voice): auto-retry transient provider errors; prime STT acronyms

Diagnosed from a live voice session (e1b9fccb90ef): three turns failed with
raw '**Error:** HTTP 502 ... hermes-{claude,codex}-broker' because the agent
pod hosting the model brokers rolled mid-conversation. The voice client
correctly refused to speak the error envelope, but it then dropped the user's
utterance and forced them to repeat it three times.

Voice: on a TRANSIENT provider error (5xx/502/'error sending request'/timeout),
conversation mode now re-runs the errored turn in place through the app's own
regenerate action (which truncates the errored turn — no duplicate user
message) and stays in Thinking so its cues cover the reconnect gap. Bounded to
MAX_TRANSIENT_RETRIES (2); a non-transient error or an exhausted budget still
drops cleanly to 'let's try that again — listening'. The raw error is never
spoken. New probe scenarios cover retry-then-recover and the bounded-then-drop
path; source contract updated.

STT: the same session mis-transcribed 'CUI' as 'cue'. Prime the default
initial_prompt with the domain acronyms the user uses (CUI, FOUO, DoD, NIST,
CMMC, FIPS, RMF, POA&M, ATO, SBU) so they bias to uppercase forms.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf

dockerfiles/hermes-jetson-stt-server.py

@@ -37,7 +37,9 @@ CACHE_DIR = Path(os.getenv("HERMES_STT_CACHE", "/cache/whisper"))
 DEFAULT_INITIAL_PROMPT = (
     "Proper nouns and names keep their capitalization and accents, for "
     "example Córdoba, Cancún, Málaga, Moscú, Москва, New York, and names "
-    "like Amy, Claude, and Hermes."
+    "like Amy, Claude, and Hermes. Technical and government acronyms are "
+    "written as uppercase letters, for example CUI, FOUO, DoD, NIST, CMMC, "
+    "FIPS, RMF, POA&M, ATO, and SBU."
 )
 INITIAL_PROMPT = os.getenv("HERMES_STT_INITIAL_PROMPT", DEFAULT_INITIAL_PROMPT).strip()
 

dockerfiles/hermes-webui-atlas-voice.js

@@ -154,6 +154,17 @@
   // callback can beat the settle re-render that stamps the answer onto
   // data-raw-text, so a transient empty read must never tear a live reply down.
   let finalizeAttempts=0;
+  // Bounded auto-retry for a TRANSIENT provider error (a broker 5xx/502 or
+  // "error sending request" reply — e.g. a model broker that blipped while its
+  // pod rolled mid-conversation). Instead of silently dropping the user's
+  // utterance and forcing them to repeat it, re-run the last user turn through
+  // the app's own regenerate path (which truncates the errored turn, so no
+  // duplicate user message), capped so a persistent failure still surfaces.
+  // Reset on any real answer and on each new user utterance.
+  let transientRetryCount=0;
+  const MAX_TRANSIENT_RETRIES=2;
+  const RECONNECT_CAPTION={en:'Reconnecting…',ru:'Переподключение…',es:'Reconectando…'};
+  const TRANSIENT_ERROR_RE=/\b(429|50[0-9])\b|error sending request|bad gateway|gateway timeout|timed? ?out|timeout|temporarily|overloaded|unavailable|connection (refused|reset|error)|reset by peer|upstream/i;
   const TTS_LANGUAGES=['en','ru','es'];
   // FIX 3: conversation-mode language chooser. Auto (default) plus the languages
   // the private voice map supports — kept in lockstep with
@@ -1122,11 +1133,94 @@
     setState('listening',statusLabel);
   }
 
+  function errorTurnText(turn){
+    // The human-visible error text of an errored assistant turn: the provider
+    // details block the renderer appends for a provider failure, else the body.
+    if(!turn||typeof turn.querySelector!=='function') return '';
+    let txt='';
+    try{
+      const details=turn.querySelector('.provider-error-details');
+      if(details&&typeof details.textContent==='string') txt=details.textContent;
+      if(!txt){
+        const body=turn.querySelector('.msg-body');
+        if(body&&typeof body.textContent==='string') txt=body.textContent;
+      }
+    }catch(_){ }
+    return txt||'';
+  }
+
+  function errorTurnIsTransient(turn){
+    // A broker/provider blip (5xx/502, "error sending request", timeout,
+    // overloaded) is recoverable by re-running the same turn; a content or
+    // policy error is not, and must never be retried into an identical failure.
+    return TRANSIENT_ERROR_RE.test(errorTurnText(turn));
+  }
+
+  function findRegenerateButton(turn){
+    // The app renders a single "regenerate" action on the last assistant turn
+    // (onclick="regenerateResponse(this)"): clicking it truncates the errored
+    // turn and re-runs the last user message with no duplicate user turn. Match
+    // the action buttons by class and filter by the onclick target so this stays
+    // correct without relying on a descendant/substring attribute selector.
+    if(!turn||typeof turn.querySelectorAll!=='function') return null;
+    let buttons;
+    try{buttons=turn.querySelectorAll('.msg-action-btn');}catch(_){return null;}
+    if(!buttons) return null;
+    for(let i=0;i<buttons.length;i+=1){
+      const button=buttons[i];
+      const onclick=(button&&typeof button.getAttribute==='function')?(button.getAttribute('onclick')||''):'';
+      if(onclick.indexOf('regenerateResponse')>=0) return button;
+    }
+    return null;
+  }
+
+  function retryTransientResponse(token,turn){
+    // Re-run the errored user turn through the app's own regenerate path and
+    // keep the conversation in Thinking — its cues cover the reconnect gap — so
+    // the retried answer streams straight into TTS. Bounded by
+    // MAX_TRANSIENT_RETRIES; a busy session or a missing regenerate control
+    // declines the retry so the caller drops cleanly. Returns true iff launched.
+    if(typeof S==='undefined'||!S||!S.session||S.busy) return false;
+    if(transientRetryCount>=MAX_TRANSIENT_RETRIES) return false;
+    const button=findRegenerateButton(turn);
+    if(!button) return false;
+    transientRetryCount+=1;
+    // Tear down only the response-side speech/observer state; do NOT resync
+    // capture (that would drop the turn and re-open the microphone).
+    thinkingSession=null;
+    thinkingTurnId='';
+    finalizeAttempts=0;
+    stopResponseObserver();
+    cancelThinkingCues();
+    cancelSpeechTurn();
+    stopPlayback();
+    pendingStitch=null;
+    const localized=normalizeSttLanguage(sessionLanguage)||'en';
+    setConversationAssistantCaption(RECONNECT_CAPTION[localized]||RECONNECT_CAPTION.en);
+    setState('thinking');
+    // Reset the answer baseline to the errored turn so the regenerated answer
+    // (which replaces it) is detected as fresh output, then arm the observer
+    // and the thinking cues before triggering the app's regenerate.
+    rememberAssistantBaseline();
+    thinkingTurnId=String(token)+'-retry-'+String(transientRetryCount);
+    startResponseObserver(token);
+    scheduleThinkingCues(token,sessionLanguage,thinkingTurnId);
+    try{button.click();}catch(_){return false;}
+    return true;
+  }
+
   function handleAssistantResponseError(token){
     // An error/system envelope (cancellation notice, provider failure) is a
     // transcript artifact, not a reply: never feed it to TTS or the reply
-    // caption, show a brief non-spoken state instead, and resynchronize
-    // capture so the next utterance starts a clean streaming turn.
+    // caption. A TRANSIENT provider blip (a broker 5xx/502) is auto-retried in
+    // place — up to MAX_TRANSIENT_RETRIES — so a momentary backend hiccup does
+    // not silently discard the user's utterance and force them to repeat it. A
+    // non-transient error, or an exhausted retry budget, shows a brief non-
+    // spoken state and resynchronizes capture for the next utterance.
+    const rows=assistantRows();
+    const turn=rows.length?assistantTurnOf(rows[rows.length-1]):null;
+    if(turn&&errorTurnIsTransient(turn)&&retryTransientResponse(token,turn)) return;
+    transientRetryCount=0;
     thinkingSession=null;
     thinkingTurnId='';
     finalizeAttempts=0;
@@ -1349,6 +1443,8 @@
     cancelVoicePreflight(turnId||captureTurnId);
     const text=String(transcript||'').trim();
     if(!text){clearSttLanguage();restartSoon(token,350);return;}
+    // A fresh user utterance starts its own transient-retry budget.
+    transientRetryCount=0;
     if(!bargeCancelPromise&&typeof S!=='undefined'&&(S.busy||S.activeStreamId)){
       // A new utterance finished while the previous model turn was still in
       // flight (continuous capture makes this a normal interruption): remember
@@ -2893,6 +2989,9 @@
       return;
     }
     finalizeAttempts=0;
+    // A real answer arrived: the provider recovered, so clear the transient
+    // retry budget for the next turn.
+    transientRetryCount=0;
     setConversationAssistantCaption(text);
     // Any answer text, even a not-yet-speakable partial clause, owns the audio
     // timeline from this point forward. A cue must never overlap or become part

testing/probes/hermes_voice_capture_probe.js

@@ -757,6 +757,41 @@ function makeHarness(options = {}) {
     runDueTimeouts,
     setAssistantReply(text) { assistantRows = [{ dataset: { rawText: text } }]; },
     setAssistantError(text) { assistantRows = [{ dataset: { rawText: text, error: '1' } }]; },
+    // A TRANSIENT provider error turn: an error envelope carrying a
+    // .provider-error-details block (so the extraction reports error), plus the
+    // last-assistant "regenerate" action button the auto-retry clicks. The
+    // button records each click so the scenario can assert the bounded retry.
+    setAssistantTransientError(text) {
+      // An errored turn is a COMPLETED turn: its stream ended, so the session is
+      // no longer busy (the same precondition the app's regenerate action needs).
+      context.S.busy = false;
+      context.S.activeStreamId = null;
+      if (context.S.session) context.S.session.active_stream_id = null;
+      const turn = buildAssistantTurn();
+      const seg = domEl('div', 'assistant-segment');
+      seg.appendChild(domEl('div', 'msg-body', null, text));
+      seg.appendChild(domEl('details', 'provider-error-details', null, text));
+      turn.blocks.appendChild(seg);
+      const foot = domEl('div', 'msg-foot');
+      const actions = domEl('span', 'msg-actions');
+      const regen = domEl('button', 'msg-action-btn', { onclick: 'regenerateResponse(this)', title: 'regenerate' });
+      regen.click = () => {
+        // Mirror the app's regenerate: truncate the errored turn and start a
+        // fresh stream. The scenario injects the next state (another error, or a
+        // recovered answer) to model whether the retried turn succeeds.
+        this._regenerateClicks = (this._regenerateClicks || 0) + 1;
+        assistantRows = [];
+        context.S.busy = true;
+        streamCounter += 1;
+        context.S.activeStreamId = `stream-${streamCounter}`;
+      };
+      actions.appendChild(regen);
+      foot.appendChild(actions);
+      turn.appendChild(foot);
+      assistantRows = [turn];
+    },
+    regenerateClicks() { return this._regenerateClicks || 0; },
+    endStream() { context.S.busy = false; context.S.activeStreamId = null; if (context.S.session) context.S.session.active_stream_id = null; },
     clearAssistantRows() { assistantRows = []; },
     // ── Interim-fold reproduction ─────────────────────────────────────────
     // Stream an interim acknowledgement as a live answer segment, then FOLD it
@@ -1018,6 +1053,62 @@ scenarios.errored_turn_is_not_spoken_and_capture_resyncs = async () => {
   };
 };
 
+// A TRANSIENT provider error (a broker 5xx/502 that blipped mid-conversation)
+// is auto-retried in place through the app's regenerate action instead of
+// dropping the user's utterance: the error is never spoken, the overlay stays
+// in Thinking (its cues cover the reconnect gap), the regenerate button is
+// clicked once, and when the retried answer arrives it is spoken normally.
+scenarios.transient_error_auto_retries_then_speaks = async () => {
+  const harness = makeHarness();
+  await harness.start();
+  await harness.silence(300);
+  await harness.speak('alpha', 1300);
+  await harness.silence(1500);
+  await harness.tick(400); // dispatched; thinking
+  const ttsBefore = harness.ttsCalls.length;
+  harness.setAssistantTransientError('**Error:** HTTP 502: error sending request for url (http://hermes-codex-broker:9003/v1/responses)');
+  await harness.tick(120); // observer sees the transient error -> one auto-retry
+  const stateAfterError = harness.state();
+  const labelAfterError = harness.elements.voiceModeLabel.textContent;
+  const clicksAfterError = harness.regenerateClicks();
+  const ttsDuringError = harness.ttsCalls.slice(ttsBefore).map((r) => r.text);
+  // The regenerated turn recovers with a real answer.
+  harness.setAssistantReply('Controlled Unclassified Information.');
+  harness.endStream();
+  await harness.tick(400);
+  return {
+    stateAfterError,
+    labelAfterError,
+    clicksAfterError,
+    ttsDuringError,
+    ttsTexts: harness.ttsCalls.map((r) => r.text),
+    state: harness.state(),
+  };
+};
+
+// The auto-retry is bounded: a provider that keeps returning a transient error
+// is retried at most MAX_TRANSIENT_RETRIES (2) times, then the turn is dropped
+// and capture resynchronizes so the failure surfaces instead of looping.
+scenarios.transient_error_retry_is_bounded = async () => {
+  const harness = makeHarness();
+  await harness.start();
+  await harness.silence(300);
+  await harness.speak('alpha', 1300);
+  await harness.silence(1500);
+  await harness.tick(400);
+  harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
+  await harness.tick(150); // retry 1 (click truncates + starts a fresh stream)
+  harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
+  await harness.tick(150); // retry 2
+  harness.setAssistantTransientError('**Error:** HTTP 503: service unavailable');
+  await harness.tick(150); // budget exhausted -> drop + resync
+  return {
+    clicks: harness.regenerateClicks(),
+    state: harness.state(),
+    label: harness.elements.voiceModeLabel.textContent,
+  };
+};
+
 // Voice barge-in appends a single-line cut marker naming the sentence that
 // was playing, so the model knows where its reply was cut off.
 scenarios.barge_cut_marker_records_spoken_tail = async () => {

testing/tests/test_hermes_stt_streaming.py

@@ -512,6 +512,10 @@ def test_initial_prompt_is_env_overridable_and_default_primes_places(monkeypatch
     assert "Córdoba" in module.DEFAULT_INITIAL_PROMPT
     assert "Москва" in module.DEFAULT_INITIAL_PROMPT
     assert "Amy" in module.DEFAULT_INITIAL_PROMPT
+    # Domain acronyms are primed so they are not misheard as common words
+    # (e.g. "CUI" transcribed as "cue").
+    assert "CUI" in module.DEFAULT_INITIAL_PROMPT
+    assert "DoD" in module.DEFAULT_INITIAL_PROMPT
     assert module.INITIAL_PROMPT == module.DEFAULT_INITIAL_PROMPT
 
     monkeypatch.setenv("HERMES_STT_INITIAL_PROMPT", "Custom prime.")

testing/tests/test_hermes_voice_capture_continuity.py

@@ -388,6 +388,31 @@ def test_errored_turn_is_never_spoken_and_capture_resyncs(probe_results):
     assert scenario["sends"] == ["alpha", "bravo"]
 
 
+def test_transient_provider_error_auto_retries_then_speaks(probe_results):
+    """A broker 5xx/502 blip is retried in place, not dropped: the error is
+    never spoken, the overlay stays in Thinking, the app's regenerate action is
+    clicked once, and the recovered answer is spoken normally."""
+    scenario = probe_results["transient_error_auto_retries_then_speaks"]
+    assert scenario["stateAfterError"] == "thinking"
+    assert scenario["labelAfterError"] == "Thinking…"
+    assert scenario["clicksAfterError"] == 1
+    # The raw provider error text is never sent to TTS.
+    assert scenario["ttsDuringError"] == []
+    assert all("502" not in text and "Error" not in text for text in scenario["ttsTexts"])
+    # The regenerated answer is spoken and the turn ends in Speaking.
+    assert "Controlled Unclassified Information." in scenario["ttsTexts"]
+    assert scenario["state"] == "speaking"
+
+
+def test_transient_provider_error_retry_is_bounded(probe_results):
+    """A provider stuck returning a transient error is retried at most twice,
+    then the turn is dropped and capture resyncs so the failure surfaces."""
+    scenario = probe_results["transient_error_retry_is_bounded"]
+    assert scenario["clicks"] == 2
+    assert scenario["state"] == "listening"
+    assert scenario["label"] == "Let’s try that again — listening"
+
+
 def test_barge_cut_marker_is_one_bounded_line(probe_results):
     scenario = probe_results["barge_cut_marker_records_spoken_tail"]
     assert scenario["ttsTexts"][0] == "The first point is ready."
@@ -418,6 +443,13 @@ def test_cut_marker_error_resync_and_speaking_cycles_source_contract():
     assert "Let’s try that again — listening" in source
     assert "segment.dataset.error==='1'" in source
     assert ".provider-error-details" in source
+    # A TRANSIENT provider error is auto-retried in place (bounded) via the app's
+    # own regenerate action, never spoken, before the drop path runs.
+    assert "function retryTransientResponse(token,turn)" in source
+    assert "function errorTurnIsTransient(turn)" in source
+    assert "const MAX_TRANSIENT_RETRIES=2" in source
+    assert "if(transientRetryCount>=MAX_TRANSIENT_RETRIES) return false" in source
+    assert "regenerateResponse" in source  # clicks the app's regenerate button
     # A cancellation with no in-flight stream never gates the send.
     assert "if(!cancellation.streamId){" in source
     # Speaking ⇄ thinking cycles inside one interim-message turn.