diff --git a/Cargo.lock b/Cargo.lock index a9eee68..9ee6b5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1642,7 +1642,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.14.17" +version = "0.14.18" dependencies = [ "anyhow", "async-stream", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.14.17" +version = "0.14.18" dependencies = [ "anyhow", "base64", @@ -1688,7 +1688,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.14.17" +version = "0.14.18" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 07198f6..0a98ee7 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.14.17" +version = "0.14.18" edition = "2024" [dependencies] diff --git a/client/src/app/uplink_media.rs b/client/src/app/uplink_media.rs index 1f0a471..c42dbad 100644 --- a/client/src/app/uplink_media.rs +++ b/client/src/app/uplink_media.rs @@ -21,6 +21,11 @@ impl LesavkaClientApp { let next = queue_stream.pop_fresh().await; if next.dropped_stale > 0 { telemetry_stream.record_stale_drop(next.dropped_stale); + warn!( + dropped_stale = next.dropped_stale, + queue_depth = next.queue_depth, + "๐ŸŽค upstream microphone queue dropped stale packets" + ); } if let Some(packet) = next.packet { telemetry_stream.record_streamed( @@ -48,6 +53,12 @@ impl LesavkaClientApp { let stats = queue_thread.push(pkt, enqueue_age); if stats.dropped_queue_full > 0 { telemetry_thread.record_queue_full_drop(stats.dropped_queue_full); + warn!( + dropped_queue_full = stats.dropped_queue_full, + queue_depth = stats.queue_depth, + enqueue_age_ms = duration_ms(enqueue_age), + "๐ŸŽค upstream microphone queue dropped the oldest packet because it was full" + ); } telemetry_thread.record_enqueue( queue_depth_u32(stats.queue_depth), @@ -103,6 +114,11 @@ impl LesavkaClientApp { let next = queue_stream.pop_fresh().await; if next.dropped_stale > 0 { telemetry_stream.record_stale_drop(next.dropped_stale); + warn!( + dropped_stale = next.dropped_stale, + queue_depth = next.queue_depth, + "๐Ÿ“ธ upstream camera queue dropped stale packets" + ); } if let Some(packet) = next.packet { telemetry_stream.record_streamed( @@ -142,6 +158,12 @@ impl LesavkaClientApp { let stats = queue.push(pkt, enqueue_age); if stats.dropped_queue_full > 0 { telemetry.record_queue_full_drop(stats.dropped_queue_full); + warn!( + dropped_queue_full = stats.dropped_queue_full, + queue_depth = stats.queue_depth, + enqueue_age_ms = duration_ms(enqueue_age), + "๐Ÿ“ธ upstream camera queue dropped the oldest frame because it was full" + ); } telemetry.record_enqueue( queue_depth_u32(stats.queue_depth), diff --git a/client/src/sync_probe/runner.rs b/client/src/sync_probe/runner.rs index 71fc5b1..18ccef6 100644 --- a/client/src/sync_probe/runner.rs +++ b/client/src/sync_probe/runner.rs @@ -70,6 +70,13 @@ async fn run_sync_probe(config: ProbeConfig) -> Result<()> { let outbound = async_stream::stream! { loop { let next = video_queue.pop_fresh().await; + if next.dropped_stale > 0 { + tracing::warn!( + dropped_stale = next.dropped_stale, + queue_depth = next.queue_depth, + "๐Ÿงช sync probe video queue dropped stale packets" + ); + } if let Some(packet) = next.packet { yield packet; continue; @@ -93,6 +100,13 @@ async fn run_sync_probe(config: ProbeConfig) -> Result<()> { let outbound = async_stream::stream! { loop { let next = audio_queue.pop_fresh().await; + if next.dropped_stale > 0 { + tracing::warn!( + dropped_stale = next.dropped_stale, + queue_depth = next.queue_depth, + "๐Ÿงช sync probe audio queue dropped stale packets" + ); + } if let Some(packet) = next.packet { sent_packets = sent_packets.saturating_add(1); if sent_packets <= 5 || sent_packets.is_multiple_of(500) { diff --git a/common/Cargo.toml b/common/Cargo.toml index 454a0e2..89e76e9 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.14.17" +version = "0.14.18" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index b85635e..4c11bbc 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -22,6 +22,7 @@ VIDEO_SIZE=${VIDEO_SIZE:-auto} VIDEO_FPS=${VIDEO_FPS:-30} VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} +REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-ffmpeg} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto} @@ -73,6 +74,7 @@ ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \ "${VIDEO_FPS}" \ "${VIDEO_FORMAT}" \ "${REMOTE_CAPTURE_STACK}" \ + "${REMOTE_PULSE_CAPTURE_TOOL}" \ "${REMOTE_PULSE_VIDEO_MODE}" \ "${REMOTE_AUDIO_SOURCE}" \ "${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \ @@ -85,9 +87,10 @@ video_size=$3 video_fps=$4 video_format=$5 remote_capture_stack=$6 -remote_pulse_video_mode=$7 -remote_audio_source=$8 -remote_audio_quiesce_user_audio=$9 +remote_pulse_capture_tool=$7 +remote_pulse_video_mode=$8 +remote_audio_source=$9 +remote_audio_quiesce_user_audio=${10} rm -f "${remote_capture}" @@ -291,36 +294,76 @@ if [[ "${capture_mode}" == "pwpipe" ]]; then "${remote_capture}" elif [[ "${capture_mode}" == "pulse" ]]; then printf 'using Pulse source: %s\n' "${pulse_source}" >&2 - case "${remote_pulse_video_mode}" in - copy) - ffmpeg -hide_banner -loglevel error -y \ - -thread_queue_size 1024 \ - "${video_args[@]}" \ - -i /dev/video0 \ - -thread_queue_size 1024 \ - -f pulse \ - -i "${pulse_source}" \ - -t "${capture_seconds}" \ - -c:v copy \ - -c:a pcm_s16le \ - "${remote_capture}" + case "${remote_pulse_capture_tool}" in + ffmpeg) + case "${remote_pulse_video_mode}" in + copy) + ffmpeg -hide_banner -loglevel error -y \ + -thread_queue_size 1024 \ + "${video_args[@]}" \ + -i /dev/video0 \ + -thread_queue_size 1024 \ + -f pulse \ + -i "${pulse_source}" \ + -t "${capture_seconds}" \ + -c:v copy \ + -c:a pcm_s16le \ + "${remote_capture}" + ;; + cfr) + ffmpeg -hide_banner -loglevel error -y \ + -thread_queue_size 1024 \ + "${video_args[@]}" \ + -i /dev/video0 \ + -thread_queue_size 1024 \ + -f pulse \ + -i "${pulse_source}" \ + -t "${capture_seconds}" \ + -vf "fps=${video_fps}" \ + -c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \ + -c:a pcm_s16le \ + "${remote_capture}" + ;; + *) + printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 + exit 64 + ;; + esac ;; - cfr) - ffmpeg -hide_banner -loglevel error -y \ - -thread_queue_size 1024 \ - "${video_args[@]}" \ - -i /dev/video0 \ - -thread_queue_size 1024 \ - -f pulse \ - -i "${pulse_source}" \ - -t "${capture_seconds}" \ - -vf "fps=${video_fps}" \ - -c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \ - -c:a pcm_s16le \ - "${remote_capture}" + gst) + case "${remote_pulse_video_mode}" in + copy) + timeout --signal=INT "$((capture_seconds + 3))" \ + gst-launch-1.0 -q -e \ + matroskamux name=mux ! filesink location="${remote_capture}" \ + v4l2src device=/dev/video0 do-timestamp=true ! \ + image/jpeg,width="${resolved_video_size%x*}",height="${resolved_video_size#*x}",framerate="${video_fps}"/1 ! \ + queue ! mux. \ + pulsesrc device="${pulse_source}" do-timestamp=true ! \ + audio/x-raw,rate=48000,channels=2 ! \ + audioconvert ! audioresample ! queue ! mux. || true + ;; + cfr) + timeout --signal=INT "$((capture_seconds + 3))" \ + gst-launch-1.0 -q -e \ + matroskamux name=mux ! filesink location="${remote_capture}" \ + v4l2src device=/dev/video0 do-timestamp=true ! \ + image/jpeg,width="${resolved_video_size%x*}",height="${resolved_video_size#*x}",framerate="${video_fps}"/1 ! \ + jpegdec ! videoconvert ! videorate ! video/x-raw,framerate="${video_fps}"/1 ! \ + jpegenc quality=90 ! image/jpeg,framerate="${video_fps}"/1 ! \ + queue ! mux. \ + pulsesrc device="${pulse_source}" do-timestamp=true ! \ + audio/x-raw,rate=48000,channels=2 ! \ + audioconvert ! audioresample ! queue ! mux. || true + ;; + *) + printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 + exit 64 + ;; + esac ;; *) - printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 + printf 'unsupported REMOTE_PULSE_CAPTURE_TOOL=%s\n' "${remote_pulse_capture_tool}" >&2 exit 64 ;; esac diff --git a/server/Cargo.toml b/server/Cargo.toml index bb3cf84..8f5e610 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.14.17" +version = "0.14.18" edition = "2024" autobins = false