hermes(voice): a stray error segment must not discard a real reply
The 'Something went wrong - listening' state with no spoken answer was a false positive: readAssistantTurn flagged the whole turn as an error if ANY segment was error-stamped - including a recovered/transient tool error or a cancellation notice from an earlier interim - and threw away the real answer that the same turn produced. Error now surfaces only when the turn yielded no spoken answer at all; a turn with real content is spoken normally. Softened the genuine-error label to the friendlier 'Let's try that again - listening'. New probe scenarios lock both: an error segment alongside an answer speaks the answer (error=false), and an error-only turn still reports the error. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
This commit is contained in:
parent
4de37f4f3a
commit
b40300efaf
@ -861,7 +861,7 @@
|
|||||||
lastSentTranscript=null;
|
lastSentTranscript=null;
|
||||||
clearBargeCancellation();
|
clearBargeCancellation();
|
||||||
setConversationAssistantCaption('');
|
setConversationAssistantCaption('');
|
||||||
resyncCapture(token,'Something went wrong — listening');
|
resyncCapture(token,'Let’s try that again — listening');
|
||||||
}
|
}
|
||||||
|
|
||||||
function assistantRows(){
|
function assistantRows(){
|
||||||
@ -998,7 +998,11 @@
|
|||||||
const value=readSegmentBody(turn);
|
const value=readSegmentBody(turn);
|
||||||
if(value&&value.trim()) parts.push(value.trim());
|
if(value&&value.trim()) parts.push(value.trim());
|
||||||
}
|
}
|
||||||
return {text:cleanForSpeech(parts.join('\n\n')),error:error};
|
// A stray error-stamped segment (a recovered tool error, or a
|
||||||
|
// cancellation notice from an earlier interim) must not discard a real
|
||||||
|
// answer: only surface the error state when the turn produced no
|
||||||
|
// spoken answer at all.
|
||||||
|
return {text:cleanForSpeech(parts.join('\n\n')),error:error&&parts.length===0};
|
||||||
}
|
}
|
||||||
|
|
||||||
function rememberAssistantBaseline(){
|
function rememberAssistantBaseline(){
|
||||||
|
|||||||
@ -399,4 +399,34 @@ const results = {};
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// (6) A turn with a recovered/transient error segment AND a real answer must
|
||||||
|
// speak the answer (error must NOT veto a turn that produced content).
|
||||||
|
{
|
||||||
|
const answer = 'Here is the real answer despite a transient tool hiccup.';
|
||||||
|
const errSeg = answerSegment(null, 'a tool call failed transiently', { idx: 1 });
|
||||||
|
errSeg.setAttribute('data-error', '1');
|
||||||
|
const turn = assistantTurn([errSeg, answerSegment(answer, answer, { idx: 2 })], { worklog: false });
|
||||||
|
setTurn(turn);
|
||||||
|
const extracted = internals.collectAssistantResponse();
|
||||||
|
results.error_segment_with_answer = {
|
||||||
|
text: extracted.text,
|
||||||
|
error: extracted.error,
|
||||||
|
speaksAnswer: extracted.text === answer,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// (7) A turn that is ONLY an error envelope (no answer) still reports error.
|
||||||
|
{
|
||||||
|
const errSeg = answerSegment(null, 'The provider returned an error.', { idx: 1 });
|
||||||
|
errSeg.setAttribute('data-error', '1');
|
||||||
|
const turn = assistantTurn([errSeg], { worklog: false });
|
||||||
|
setTurn(turn);
|
||||||
|
const extracted = internals.collectAssistantResponse();
|
||||||
|
results.error_only_reports_error = {
|
||||||
|
text: extracted.text,
|
||||||
|
isEmpty: extracted.text === '',
|
||||||
|
error: extracted.error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
process.stdout.write(JSON.stringify(results, null, 2) + '\n');
|
process.stdout.write(JSON.stringify(results, null, 2) + '\n');
|
||||||
|
|||||||
@ -182,7 +182,7 @@ def test_two_word_young_utterance_keeps_the_hold(probe_results):
|
|||||||
def test_errored_turn_is_never_spoken_and_capture_resyncs(probe_results):
|
def test_errored_turn_is_never_spoken_and_capture_resyncs(probe_results):
|
||||||
scenario = probe_results["errored_turn_is_not_spoken_and_capture_resyncs"]
|
scenario = probe_results["errored_turn_is_not_spoken_and_capture_resyncs"]
|
||||||
assert scenario["stateAfterError"] == "listening"
|
assert scenario["stateAfterError"] == "listening"
|
||||||
assert scenario["labelAfterError"] == "Something went wrong — listening"
|
assert scenario["labelAfterError"] == "Let’s try that again — listening"
|
||||||
assert scenario["ttsDuringError"] == 0
|
assert scenario["ttsDuringError"] == 0
|
||||||
assert scenario["freshSessions"] >= 1
|
assert scenario["freshSessions"] >= 1
|
||||||
# The follow-up utterance is sent alone: no stitch with the errored turn.
|
# The follow-up utterance is sent alone: no stitch with the errored turn.
|
||||||
@ -216,7 +216,7 @@ def test_cut_marker_error_resync_and_speaking_cycles_source_contract():
|
|||||||
# Error envelopes are detected structurally and trigger a capture resync.
|
# Error envelopes are detected structurally and trigger a capture resync.
|
||||||
assert "function handleAssistantResponseError(token)" in source
|
assert "function handleAssistantResponseError(token)" in source
|
||||||
assert "function resyncCapture(token,statusLabel)" in source
|
assert "function resyncCapture(token,statusLabel)" in source
|
||||||
assert "'Something went wrong — listening'" in source
|
assert "Let’s try that again — listening" in source
|
||||||
assert "segment.dataset.error==='1'" in source
|
assert "segment.dataset.error==='1'" in source
|
||||||
assert ".provider-error-details" in source
|
assert ".provider-error-details" in source
|
||||||
# A cancellation with no in-flight stream never gates the send.
|
# A cancellation with no in-flight stream never gates the send.
|
||||||
|
|||||||
@ -45,6 +45,18 @@ def results() -> dict:
|
|||||||
return json.loads(completed.stdout)
|
return json.loads(completed.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_segment_does_not_discard_a_real_answer(results):
|
||||||
|
scenario = results["error_segment_with_answer"]
|
||||||
|
assert scenario["speaksAnswer"] is True, "a stray error segment discarded the real answer"
|
||||||
|
assert scenario["error"] is False, "a turn with a real answer must not surface the error state"
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_only_turn_still_reports_error(results):
|
||||||
|
scenario = results["error_only_reports_error"]
|
||||||
|
assert scenario["isEmpty"] is True
|
||||||
|
assert scenario["error"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_caption_is_answer_body_only(results):
|
def test_caption_is_answer_body_only(results):
|
||||||
"""Symptom 1: caption/TTS text must be the answer body, never row chrome."""
|
"""Symptom 1: caption/TTS text must be the answer body, never row chrome."""
|
||||||
scenario = results["finalized_multi_segment"]
|
scenario = results["finalized_multi_segment"]
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user