From e17464e1f9c5f5cbd7eee056cb0136dab7ca31d4 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 4 May 2026 14:05:55 -0300 Subject: [PATCH] test(server-rc): stabilize pulse capture harness --- Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- .../manual/run_server_to_rc_mode_matrix.sh | 3 + scripts/manual/run_upstream_av_sync.sh | 96 +++++++++++-------- server/Cargo.toml | 2 +- .../client_manual_sync_script_contract.rs | 20 ++-- 7 files changed, 76 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a15460c..482ec1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.15" +version = "0.19.16" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.15" +version = "0.19.16" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.15" +version = "0.19.16" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index f26b499..31b7d46 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.15" +version = "0.19.16" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 386e937..8d5beb0 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.15" +version = "0.19.16" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh index 9881e54..eb4dd97 100755 --- a/scripts/manual/run_server_to_rc_mode_matrix.sh +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -81,6 +81,7 @@ REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} +REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0} LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0} STAMP="$(date +%Y%m%d-%H%M%S)" @@ -963,6 +964,7 @@ echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}" echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}" echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" +echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}" echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}" echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s" @@ -1000,6 +1002,7 @@ for mode in "${modes[@]}"; do REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \ REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \ REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \ + REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \ PROBE_PREBUILD=0 \ VIDEO_SIZE="${width}x${height}" \ diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index 249f22f..496178f 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -19,7 +19,8 @@ LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https} LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server} PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20} PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4} -PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + 20))} +PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20} +PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))} PROBE_PULSE_PERIOD_MS=${PROBE_PULSE_PERIOD_MS:-1000} PROBE_PULSE_WIDTH_MS=${PROBE_PULSE_WIDTH_MS:-120} PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2} @@ -30,7 +31,7 @@ LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DEL # VIDIOC_STREAMON if the camera is starved during pre-roll. LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0} TAIL_SECONDS=${TAIL_SECONDS:-2} -CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))} +CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))} LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp} REMOTE_VIDEO_DEVICE=${REMOTE_VIDEO_DEVICE:-auto} VIDEO_SIZE=${VIDEO_SIZE:-auto} @@ -41,6 +42,7 @@ REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto} +REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0} REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0} ANALYSIS_NORMALIZE=${ANALYSIS_NORMALIZE:-0} ANALYSIS_SCALE_WIDTH=${ANALYSIS_SCALE_WIDTH:-1280} @@ -533,11 +535,11 @@ preflight_server_path() { "${REMOTE_EXPECT_UVC_HEIGHT}" \ "${REMOTE_EXPECT_UVC_FPS}" <<'REMOTE_PREFLIGHT' set -euo pipefail -expect_cam_output=$1 -expect_uvc_codec=$2 -expect_uvc_width=$3 -expect_uvc_height=$4 -expect_uvc_fps=$5 +expect_cam_output=${1:-} +expect_uvc_codec=${2:-} +expect_uvc_width=${3:-} +expect_uvc_height=${4:-} +expect_uvc_fps=${5:-} read_env_value() { local key=$1 @@ -1978,6 +1980,7 @@ ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \ "${REMOTE_PULSE_VIDEO_MODE}" \ "${REMOTE_AUDIO_SOURCE}" \ "${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \ + "${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ "${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS}" \ > >(tee "${LOCAL_CAPTURE_LOG}") \ 2> >(tee -a "${LOCAL_CAPTURE_LOG}" >&2) <<'REMOTE_CAPTURE_SCRIPT' & @@ -1993,7 +1996,8 @@ remote_pulse_capture_tool=$8 remote_pulse_video_mode=$9 remote_audio_source=${10} remote_audio_quiesce_user_audio=${11} -remote_capture_preroll_discard_seconds=${12} +remote_capture_allow_alsa_fallback=${12} +remote_capture_preroll_discard_seconds=${13} rm -f "${remote_capture}" @@ -2252,7 +2256,7 @@ raise SystemExit(1) ' } -capture_mode="alsa" +capture_mode="" alsa_audio_dev="hw:3,0" pulse_source="" pw_audio_target="" @@ -2262,21 +2266,23 @@ case "${remote_capture_stack}" in if [[ "${remote_audio_source}" == "auto" ]]; then if pulse_source="$(resolve_pulse_source)"; then capture_mode="pulse" - elif alsa_audio_dev="$(resolve_alsa_audio_device)"; then - capture_mode="alsa" - printf 'PipeWire Lesavka source not found; falling back to ALSA device %s\n' "${alsa_audio_dev}" >&2 elif command -v pw-record >/dev/null 2>&1 \ && command -v pw-v4l2 >/dev/null 2>&1 \ && pw_audio_target="$(resolve_pw_audio_target)"; then capture_mode="pwpipe" + elif [[ "${remote_capture_allow_alsa_fallback}" == "1" ]] && alsa_audio_dev="$(resolve_alsa_audio_device)"; then + capture_mode="alsa" + printf 'PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device %s\n' "${alsa_audio_dev}" >&2 else - printf 'Lesavka audio source not found in PipeWire or ALSA; capture host does not currently expose the gadget microphone.\n' >&2 + printf 'Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.\n' >&2 + printf 'Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.\n' >&2 exit 64 fi elif [[ "${remote_audio_source}" == pulse:* ]]; then capture_mode="pulse" pulse_source="${remote_audio_source#pulse:}" elif [[ "${remote_audio_source}" == alsa:* ]]; then + capture_mode="alsa" alsa_audio_dev="${remote_audio_source#alsa:}" else printf 'unsupported REMOTE_AUDIO_SOURCE=%s\n' "${remote_audio_source}" >&2 @@ -2452,11 +2458,8 @@ elif [[ "${capture_mode}" == "pulse" ]]; then -thread_queue_size 1024 \ -f pulse \ -i "${pulse_source}" \ - -f lavfi \ - -i anullsrc=channel_layout=stereo:sample_rate=48000 \ - -filter_complex "[1:a][2:a]amix=inputs=2:duration=longest:dropout_transition=0[aout]" \ -map 0:v:0 \ - -map "[aout]" \ + -map 1:a:0 \ -t "${capture_seconds}" \ -c:v copy \ -c:a pcm_s16le \ @@ -2470,11 +2473,8 @@ elif [[ "${capture_mode}" == "pulse" ]]; then -thread_queue_size 1024 \ -f pulse \ -i "${pulse_source}" \ - -f lavfi \ - -i anullsrc=channel_layout=stereo:sample_rate=48000 \ - -filter_complex "[1:a][2:a]amix=inputs=2:duration=longest:dropout_transition=0[aout]" \ -map 0:v:0 \ - -map "[aout]" \ + -map 1:a:0 \ -t "${capture_seconds}" \ -vf "fps=${resolved_video_fps}" \ -c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \ @@ -2500,13 +2500,8 @@ elif [[ "${capture_mode}" == "pulse" ]]; then v4l2src device="${resolved_video_device}" do-timestamp=true ! \ ${gst_source_caps} ! \ queue ! mux. \ - audiotestsrc wave=silence is-live=true samplesperbuffer=480 ! \ - audio/x-raw,rate=48000,channels=2 ! \ - queue ! mix. \ pulsesrc device="${pulse_source}" do-timestamp=true ! \ audio/x-raw,rate=48000,channels=2 ! \ - audioconvert ! audioresample ! queue ! mix. \ - audiomixer name=mix ! \ audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \ queue ! mux. || true ;; @@ -2521,13 +2516,8 @@ elif [[ "${capture_mode}" == "pulse" ]]; then x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \ h264parse ! \ queue ! mux. \ - audiotestsrc wave=silence is-live=true samplesperbuffer=480 ! \ - audio/x-raw,rate=48000,channels=2 ! \ - queue ! mix. \ pulsesrc device="${pulse_source}" do-timestamp=true ! \ audio/x-raw,rate=48000,channels=2 ! \ - audioconvert ! audioresample ! queue ! mix. \ - audiomixer name=mix ! \ audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \ queue ! mux. || true ;; @@ -2543,17 +2533,39 @@ elif [[ "${capture_mode}" == "pulse" ]]; then ;; esac else - run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \ - -thread_queue_size 1024 \ - "${video_args[@]}" \ - -i "${resolved_video_device}" \ - -thread_queue_size 1024 \ - -f alsa -ac 2 -ar 48000 \ - -i "${alsa_audio_dev}" \ - -t "${capture_seconds}" \ - -c:v ffv1 -level 3 -g 1 \ - -c:a pcm_s16le \ - "${remote_capture}" + case "${remote_pulse_video_mode}" in + copy) + run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \ + -thread_queue_size 1024 \ + "${video_args[@]}" \ + -i "${resolved_video_device}" \ + -thread_queue_size 1024 \ + -f alsa -ac 2 -ar 48000 \ + -i "${alsa_audio_dev}" \ + -t "${capture_seconds}" \ + -c:v copy \ + -c:a pcm_s16le \ + "${remote_capture}" + ;; + cfr) + run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \ + -thread_queue_size 1024 \ + "${video_args[@]}" \ + -i "${resolved_video_device}" \ + -thread_queue_size 1024 \ + -f alsa -ac 2 -ar 48000 \ + -i "${alsa_audio_dev}" \ + -t "${capture_seconds}" \ + -vf "fps=${resolved_video_fps}" \ + -c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \ + -c:a pcm_s16le \ + "${remote_capture}" + ;; + *) + printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 + exit 64 + ;; + esac fi REMOTE_CAPTURE_SCRIPT capture_pid=$! diff --git a/server/Cargo.toml b/server/Cargo.toml index 97a578e..13c4b15 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.15" +version = "0.19.16" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index ae83c52..19cca79 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -139,11 +139,14 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { "\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"", "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"", "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}", + "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}", "output_delay_calibration_json", "direct UVC/UAC output-delay calibration", "calibration-save-default", "LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}", - "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + 20))}", + "PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}", + "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}", + "CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}", "ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-1}", "compute_analysis_window_arg", "analyzer timeline window:", @@ -159,13 +162,13 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { "grep -q \"${CAPTURE_READY_MARKER}\"", "Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.", "resolve_alsa_audio_device", - "PipeWire Lesavka source not found; falling back to ALSA device", - "Lesavka audio source not found in PipeWire or ALSA; capture host does not currently expose the gadget microphone.", + "PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device", + "Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.", + "Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.", "discarding %ss of post-enumeration capture before probe", - "audiotestsrc wave=silence is-live=true samplesperbuffer=480", - "audiomixer name=mix", - "anullsrc=channel_layout=stereo:sample_rate=48000", - "amix=inputs=2:duration=longest", + "using Pulse source:", + "-f pulse", + "-map 1:a:0", "artifact_dir: ${LOCAL_REPORT_DIR}", "events_csv: ${LOCAL_EVENTS_CSV}", "server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}", @@ -238,6 +241,7 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}", "video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}", "audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}", + "pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL}", "fast runtime env updated: CAM_OUTPUT=uvc", "cycling UVC gadget descriptors", "lesavka-core reconfigure log:", @@ -262,6 +266,8 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "calibration:", "REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"", "REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"", + "REMOTE_CAPTURE_STACK=\"${REMOTE_CAPTURE_STACK}\"", + "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=\"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"", "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=\"${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"", "PROBE_PREBUILD=0", "VIDEO_SIZE=\"${width}x${height}\"",