b40300efaf
The 'Something went wrong - listening' state with no spoken answer was a false positive: readAssistantTurn flagged the whole turn as an error if ANY segment was error-stamped - including a recovered/transient tool error or a cancellation notice from an earlier interim - and threw away the real answer that the same turn produced. Error now surfaces only when the turn yielded no spoken answer at all; a turn with real content is spoken normally. Softened the genuine-error label to the friendlier 'Let's try that again - listening'. New probe scenarios lock both: an error segment alongside an answer speaks the answer (error=false), and an error-only turn still reports the error. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
351 lines
16 KiB
Python
351 lines
16 KiB
Python
"""Continuous-capture contracts: no first-word clipping, stitching, overlay.
|
||
|
||
The scenarios drive the real ``atlas-voice.js`` against a stub streaming STT
|
||
WebSocket whose VAD, speculative EOS-freeze, resume and commit semantics
|
||
mirror ``hermes-jetson-stt-server.py``. Words are encoded as distinct PCM
|
||
amplitudes, so the transcript of a commit is exactly the words whose audio
|
||
survived endpointing — the live "only the first word was sent" regression is
|
||
directly observable in the composer text the probe records.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import shutil
|
||
import subprocess
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
VOICE_SCRIPT = ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js"
|
||
PROBE = ROOT / "testing" / "probes" / "hermes_voice_capture_probe.js"
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def probe_results() -> dict:
|
||
node = shutil.which("node")
|
||
if not node:
|
||
pytest.skip("node is required to drive the capture continuity contract")
|
||
completed = subprocess.run(
|
||
[node, str(PROBE), str(VOICE_SCRIPT)],
|
||
check=False,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=180,
|
||
)
|
||
assert completed.returncode == 0, completed.stderr
|
||
return json.loads(completed.stdout)
|
||
|
||
|
||
def test_multiword_utterance_with_interword_pause_is_complete(probe_results):
|
||
scenario = probe_results["multiword_utterance_with_interword_pause"]
|
||
assert scenario["sends"] == ["alpha bravo"]
|
||
assert scenario["state"] == "thinking"
|
||
|
||
|
||
def test_thinking_pause_after_first_word_does_not_split(probe_results):
|
||
"""A >1100ms pause right after the first word stays inside one utterance."""
|
||
scenario = probe_results["thinking_pause_after_first_word_does_not_split"]
|
||
assert scenario["sends"] == ["alpha bravo"]
|
||
# The pause froze a first-word EOS snapshot on the server; resumed speech
|
||
# must have cleared it before the commit.
|
||
events = scenario["serverEvents"][0]
|
||
assert "resume-clear" in events or "recover-by-rms" in events
|
||
|
||
|
||
def test_barge_in_stitches_regardless_of_partial_assistant_output(probe_results):
|
||
scenario = probe_results["second_utterance_during_response_is_complete"]
|
||
assert scenario["firstSends"] == ["alpha"]
|
||
# Assistant text was already visible (and being spoken) when the user
|
||
# talked over the response: the interrupted thought and the follow-up form
|
||
# one stitched message, and the cut marker names the sentence that was
|
||
# playing so the model knows where its reply stopped being heard.
|
||
assert scenario["sends"] == [
|
||
"alpha",
|
||
"alpha bravo charlie\n[voice interruption: you were cut off after "
|
||
'"Partial answer already visible."]',
|
||
]
|
||
|
||
|
||
def test_normal_completion_clears_stitch_and_mic_stays_hot(probe_results):
|
||
scenario = probe_results["back_to_back_utterances_stay_hot"]
|
||
assert scenario["sendsAfterFirst"] == ["alpha"]
|
||
assert scenario["sends"] == ["alpha", "bravo"]
|
||
# The whole session runs on one getUserMedia lease.
|
||
assert scenario["micAcquisitions"] == 1
|
||
|
||
|
||
def test_conversation_overlay_lifecycle(probe_results):
|
||
scenario = probe_results["conversation_overlay_lifecycle"]
|
||
assert scenario["overlayPresent"] is True
|
||
assert scenario["role"] == "dialog"
|
||
assert scenario["ariaModal"] == "true"
|
||
assert scenario["captionsLive"] == "polite"
|
||
assert scenario["listeningState"] == "listening"
|
||
assert scenario["thinkingState"] in {"thinking", "speaking"}
|
||
assert scenario["userCaption"] == "alpha"
|
||
assert scenario["assistantCaption"] == "A visible answer."
|
||
# Mute drives the real capture track and is fully reversible.
|
||
assert scenario["mutedPressed"] == "true"
|
||
assert scenario["mutedTracks"] == [False]
|
||
assert scenario["unmutedTracks"] == [True]
|
||
# Both exit paths remove the overlay completely and end hands-free mode.
|
||
assert scenario["removedOnExit"] is True
|
||
assert scenario["inactiveAfterExit"] is True
|
||
assert scenario["removedOnEscape"] is True
|
||
|
||
|
||
def test_overlay_source_contract():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
# Lazily created, fully removed, storage-free, single-capture overlay.
|
||
assert "function openConversationOverlay()" in source
|
||
assert "function removeConversationOverlay()" in source
|
||
assert source.count("navigator.mediaDevices.getUserMedia(") == 1
|
||
overlay_region = source.split("Conversation mode overlay", 1)[1].split(
|
||
"function clearErrorTimer", 1
|
||
)[0]
|
||
assert "localStorage" not in overlay_region
|
||
assert "track.enabled=!conversation.muted" in overlay_region
|
||
assert "'aria-label':'Voice conversation'" in overlay_region
|
||
assert "'aria-live':'polite'" in overlay_region
|
||
assert "event.key==='Escape'" in overlay_region
|
||
assert "event.key==='Tab'" in overlay_region
|
||
|
||
# Orb styling: breathing, level-driven, playback-driven, reduced-motion.
|
||
assert ".voice-conversation" in css
|
||
assert "--conversation-level" in css
|
||
assert "voice-conversation-breathe" in css
|
||
assert ".voice-conversation.is-playing .voice-conversation-orb-halo" in css
|
||
reduced = css.split("@media (prefers-reduced-motion: reduce)", 1)[1]
|
||
assert ".voice-conversation *" in reduced
|
||
|
||
|
||
def test_endpointing_and_streaming_hardening_source_contract():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
|
||
# Young utterances hold their endpoint past a thinking pause — unless the
|
||
# streaming partial already reads as a plausibly complete utterance
|
||
# (>=3 words or terminal punctuation), which endpoints at the base window.
|
||
assert "const VAD_EARLY_SILENCE_MS=1800" in source
|
||
assert "const VAD_COMMITTED_SPEECH_MS=1200" in source
|
||
assert "latestPartial:function(){return lastPartialText;}" in source
|
||
assert "const partialComplete=partialWords>=3||(partialWords>0" in source
|
||
assert (
|
||
"const endpointSilenceMs=(speechMs<VAD_COMMITTED_SPEECH_MS&&!partialComplete)"
|
||
"?Math.max(silenceMs,VAD_EARLY_SILENCE_MS):silenceMs" in source
|
||
)
|
||
# Resume always reaches the wire after a server-length silence gap.
|
||
assert "const SERVER_EOS_SILENCE_MS=650" in source
|
||
assert "speechGapMs>=SERVER_EOS_SILENCE_MS" in source
|
||
assert "if(settled||committed||!speculative) return;" not in source.split(
|
||
"resume:function()", 1
|
||
)[1].split("speculate:function()", 1)[0]
|
||
# No speculative Whisper pass on a one-word fragment.
|
||
assert "const SPECULATE_MIN_SPEECH_MS=700" in source
|
||
assert "speechMs>=SPECULATE_MIN_SPEECH_MS" in source
|
||
# Echo residue can no longer ratchet the onset threshold above speech.
|
||
assert "Math.min(rms,speechThreshold)*0.06" in source
|
||
|
||
|
||
def test_stitch_gate_ignores_partial_assistant_output():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
region = source.split("function recordPendingStitch()", 1)[1].split(
|
||
"async function sendTranscript", 1
|
||
)[0]
|
||
assert "currentAssistantText()" not in region
|
||
assert "pendingStitch={text:sent.text,at:Date.now(),cut:cut}" in region
|
||
# Normal completion still clears the stitch context.
|
||
assert (
|
||
"if(isFinal){\n // The completion callback marks a normally "
|
||
"completed response" in source
|
||
)
|
||
|
||
|
||
def test_three_word_partial_endpoints_at_base_silence(probe_results):
|
||
scenario = probe_results["three_word_partial_endpoints_at_base_silence"]
|
||
assert scenario["sends"] == ["alpha bravo charlie"]
|
||
assert scenario["state"] == "thinking"
|
||
|
||
|
||
def test_two_word_young_utterance_keeps_the_hold(probe_results):
|
||
scenario = probe_results["two_word_young_utterance_keeps_the_hold"]
|
||
assert scenario["sendsEarly"] == []
|
||
assert scenario["sends"] == ["alpha bravo"]
|
||
|
||
|
||
def test_errored_turn_is_never_spoken_and_capture_resyncs(probe_results):
|
||
scenario = probe_results["errored_turn_is_not_spoken_and_capture_resyncs"]
|
||
assert scenario["stateAfterError"] == "listening"
|
||
assert scenario["labelAfterError"] == "Let’s try that again — listening"
|
||
assert scenario["ttsDuringError"] == 0
|
||
assert scenario["freshSessions"] >= 1
|
||
# The follow-up utterance is sent alone: no stitch with the errored turn.
|
||
assert scenario["sends"] == ["alpha", "bravo"]
|
||
|
||
|
||
def test_barge_cut_marker_is_one_bounded_line(probe_results):
|
||
scenario = probe_results["barge_cut_marker_records_spoken_tail"]
|
||
assert scenario["ttsTexts"][0] == "The first point is ready."
|
||
assert scenario["sends"][1] == (
|
||
"alpha bravo\n[voice interruption: you were cut off after "
|
||
'"The first point is ready."]'
|
||
)
|
||
marker = scenario["sends"][1].split("\n", 1)[1]
|
||
assert "\n" not in marker
|
||
|
||
|
||
def test_sticky_language_biases_next_stt_session(probe_results):
|
||
scenario = probe_results["sticky_language_biases_next_stt_session"]
|
||
assert scenario["startLanguages"][0] == "auto"
|
||
assert "en" in scenario["startLanguages"]
|
||
|
||
|
||
def test_cut_marker_error_resync_and_speaking_cycles_source_contract():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
|
||
# Cut marker: exactly one appended line, tail bounded to 120 characters.
|
||
assert "function voiceCutMarker(cut)" in source
|
||
assert "tail.length>120?tail.slice(tail.length-120):tail" in source
|
||
assert '\\n[voice interruption: you were cut off after "' in source
|
||
# Error envelopes are detected structurally and trigger a capture resync.
|
||
assert "function handleAssistantResponseError(token)" in source
|
||
assert "function resyncCapture(token,statusLabel)" in source
|
||
assert "Let’s try that again — listening" in source
|
||
assert "segment.dataset.error==='1'" in source
|
||
assert ".provider-error-details" in source
|
||
# A cancellation with no in-flight stream never gates the send.
|
||
assert "if(!cancellation.streamId){" in source
|
||
# Speaking ⇄ thinking cycles inside one interim-message turn.
|
||
assert "function scheduleSpeakingIdleFallback(turn,session)" in source
|
||
assert "if(state==='speaking') setState('thinking')" in source
|
||
assert "function playbackAudible()" in source
|
||
# Body-only scraping: never the avatar letter, author name or status chip.
|
||
assert "querySelectorAll('.msg-body')" in source
|
||
assert "function readSegmentBody(segment)" in source
|
||
assert "function readAssistantTurn(turn)" in source
|
||
|
||
|
||
def test_caption_regions_are_bounded_scrollable_and_follow_tail():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
assert "function updateCaptionRegion(element,text,limit)" in source
|
||
assert "function attachCaptionScroll(element)" in source
|
||
assert "element.dataset.follow=gap<=24?'1':'0'" in source
|
||
for token in (
|
||
"overflow-y: auto",
|
||
"overscroll-behavior: contain",
|
||
"-webkit-overflow-scrolling: touch",
|
||
"touch-action: pan-y",
|
||
"max-height: 22vh",
|
||
"max-height: 38vh",
|
||
"mask-image",
|
||
):
|
||
assert token in css
|
||
|
||
|
||
def test_reply_language_stickiness_source_contract():
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
|
||
assert "let sessionLanguage=''" in source
|
||
# A user-forced conversation-mode language (FIX 3) overrides the sticky
|
||
# auto-detected hint for the streaming STT session.
|
||
assert "language:forcedLanguage||sessionLanguage||'auto'" in source
|
||
assert "function strongReplyLanguage(text)" in source
|
||
assert "function detectReplyLanguage(text)" in source
|
||
assert "scheduleThinkingCues(token,language||sessionLanguage,thinkingTurnId)" in source
|
||
# The sticky language resets with each hands-free session.
|
||
assert source.count("sessionLanguage='';") >= 2
|
||
|
||
|
||
def test_conversation_language_override_forces_stt_and_voice(probe_results):
|
||
"""FIX 3: picking Russian in the overlay forces BOTH the streaming STT hint
|
||
and the reply TTS voice for the session (overriding auto-detection), and Auto
|
||
releases the override."""
|
||
scenario = probe_results["language_override_forces_stt_and_voice"]
|
||
assert scenario["overlayPresent"] is True
|
||
assert scenario["ruItemPresent"] is True
|
||
# Selection state is reflected accessibly (menuitemradio aria-checked).
|
||
assert scenario["forcedChecked"] == "true"
|
||
assert scenario["autoChecked"] == "false"
|
||
assert scenario["btnForced"] is True
|
||
# Every STT session opened after the override carries the forced hint, and an
|
||
# ENGLISH reply is still spoken by the Russian voice — auto-detection is
|
||
# fully overridden.
|
||
assert scenario["startLanguages"][1:] == ["ru", "ru", "ru"]
|
||
assert scenario["ttsLanguages"] == ["ru", "ru"]
|
||
# Auto releases the override.
|
||
assert scenario["releasedChecked"] == "true"
|
||
assert scenario["btnForcedAfterAuto"] is False
|
||
|
||
|
||
def test_conversation_language_selector_source_and_style_contract():
|
||
"""FIX 3: the corner language control is present, accessible, session-only
|
||
(no localStorage), and lists Auto + the server voice-map languages."""
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(encoding="utf-8")
|
||
|
||
assert "let forcedLanguage=''" in source
|
||
assert "const CONVERSATION_LANGUAGES=[" in source
|
||
# Auto + en/es/ru mirror hermes-jetson-tts-server.py LANGUAGE_VOICE_MAP.
|
||
for code in ("{code:'',label:'Auto'", "code:'en'", "code:'es'", "code:'ru'"):
|
||
assert code in source
|
||
assert "function selectConversationLanguage(code)" in source
|
||
assert "aria-haspopup" in source and "'aria-expanded':'false'" in source
|
||
assert "role:'menuitemradio'" in source
|
||
# Forced language overrides both STT (start hint) and the reply voice.
|
||
assert "language:forcedLanguage||sessionLanguage||'auto'" in source
|
||
assert "const resolved=forcedLanguage||strongReplyLanguage(text)" in source
|
||
# Session-only: the forced language never touches localStorage.
|
||
assert "localStorage" not in source.split("CONVERSATION_LANGUAGES", 1)[1].split("function openConversationOverlay", 1)[0]
|
||
# Escape peels the menu before exiting; the language button joins the trap.
|
||
assert "if(conversation.langMenu&&!conversation.langMenu.hidden){closeLanguageMenu(true);return;}" in source
|
||
assert "[conversation.langBtn,conversation.muteBtn,conversation.exitBtn]" in source
|
||
for token in (".voice-conversation-lang-btn", ".voice-conversation-lang-menu", "menuitemradio"):
|
||
pass # menu roles live in JS; assert CSS hooks below
|
||
for token in (".voice-conversation-lang", ".voice-conversation-lang-btn", ".voice-conversation-lang-menu", ".voice-conversation-lang-item"):
|
||
assert token in css
|
||
|
||
|
||
def test_conversation_orb_hermes_mark_source_and_style_contract():
|
||
"""FIX 2: the Hermes caduceus mark is embedded in the orb as a static,
|
||
low-opacity watermark that respects reduced motion and never animates."""
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(encoding="utf-8")
|
||
|
||
assert "const HERMES_MARK_SVG=" in source
|
||
assert 'fill-rule="evenodd"' in source # the real favicon caduceus path
|
||
assert "'voice-conversation-orb-mark'" in source
|
||
assert "orbMark.innerHTML=HERMES_MARK_SVG" in source
|
||
assert ".voice-conversation-orb-mark" in css
|
||
# Monochrome via currentColor at low opacity; carries no animation of its own.
|
||
assert "color: rgba(233, 244, 255, 0.9)" in css
|
||
mark_rule = css.split(".voice-conversation-orb-mark {", 1)[1].split("}", 1)[0]
|
||
assert "animation" not in mark_rule
|
||
assert "opacity: 0.16" in mark_rule
|
||
|
||
|
||
def test_conversation_thinking_working_affordance_contract():
|
||
"""FIX 4: a delayed 'working…' affordance during silent Thinking, CSS-driven
|
||
and reduced-motion aware, with no fabricated spoken acknowledgement."""
|
||
source = VOICE_SCRIPT.read_text(encoding="utf-8")
|
||
css = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.css").read_text(encoding="utf-8")
|
||
|
||
assert "const AWAITING_AFFORDANCE_MS=2500" in source
|
||
assert "function updateAwaitingAffordance(next)" in source
|
||
assert "conversation.root.classList.add('is-awaiting')" in source
|
||
# The affordance disarms the instant any reply text arrives.
|
||
assert "if(text&&String(text).trim()) clearAwaitingAffordance();" in source
|
||
assert ".voice-conversation.is-awaiting" in css
|
||
# Reduced motion suppresses every overlay animation, including this one.
|
||
rm_block = css.split("@media (prefers-reduced-motion: reduce)", 1)[1]
|
||
assert "animation: none !important" in rm_block
|