chore(sync): add host capture debug lanes

This commit is contained in:
Brad Stein 2026-04-27 10:24:53 -03:00
parent 7793f1eb41
commit 67f4cd156e
7 changed files with 115 additions and 36 deletions

6
Cargo.lock generated
View File

@ -1642,7 +1642,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "lesavka_client" name = "lesavka_client"
version = "0.14.17" version = "0.14.18"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -1676,7 +1676,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_common" name = "lesavka_common"
version = "0.14.17" version = "0.14.18"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",
@ -1688,7 +1688,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_server" name = "lesavka_server"
version = "0.14.17" version = "0.14.18"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",

View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package] [package]
name = "lesavka_client" name = "lesavka_client"
version = "0.14.17" version = "0.14.18"
edition = "2024" edition = "2024"
[dependencies] [dependencies]

View File

@ -21,6 +21,11 @@ impl LesavkaClientApp {
let next = queue_stream.pop_fresh().await; let next = queue_stream.pop_fresh().await;
if next.dropped_stale > 0 { if next.dropped_stale > 0 {
telemetry_stream.record_stale_drop(next.dropped_stale); telemetry_stream.record_stale_drop(next.dropped_stale);
warn!(
dropped_stale = next.dropped_stale,
queue_depth = next.queue_depth,
"🎤 upstream microphone queue dropped stale packets"
);
} }
if let Some(packet) = next.packet { if let Some(packet) = next.packet {
telemetry_stream.record_streamed( telemetry_stream.record_streamed(
@ -48,6 +53,12 @@ impl LesavkaClientApp {
let stats = queue_thread.push(pkt, enqueue_age); let stats = queue_thread.push(pkt, enqueue_age);
if stats.dropped_queue_full > 0 { if stats.dropped_queue_full > 0 {
telemetry_thread.record_queue_full_drop(stats.dropped_queue_full); telemetry_thread.record_queue_full_drop(stats.dropped_queue_full);
warn!(
dropped_queue_full = stats.dropped_queue_full,
queue_depth = stats.queue_depth,
enqueue_age_ms = duration_ms(enqueue_age),
"🎤 upstream microphone queue dropped the oldest packet because it was full"
);
} }
telemetry_thread.record_enqueue( telemetry_thread.record_enqueue(
queue_depth_u32(stats.queue_depth), queue_depth_u32(stats.queue_depth),
@ -103,6 +114,11 @@ impl LesavkaClientApp {
let next = queue_stream.pop_fresh().await; let next = queue_stream.pop_fresh().await;
if next.dropped_stale > 0 { if next.dropped_stale > 0 {
telemetry_stream.record_stale_drop(next.dropped_stale); telemetry_stream.record_stale_drop(next.dropped_stale);
warn!(
dropped_stale = next.dropped_stale,
queue_depth = next.queue_depth,
"📸 upstream camera queue dropped stale packets"
);
} }
if let Some(packet) = next.packet { if let Some(packet) = next.packet {
telemetry_stream.record_streamed( telemetry_stream.record_streamed(
@ -142,6 +158,12 @@ impl LesavkaClientApp {
let stats = queue.push(pkt, enqueue_age); let stats = queue.push(pkt, enqueue_age);
if stats.dropped_queue_full > 0 { if stats.dropped_queue_full > 0 {
telemetry.record_queue_full_drop(stats.dropped_queue_full); telemetry.record_queue_full_drop(stats.dropped_queue_full);
warn!(
dropped_queue_full = stats.dropped_queue_full,
queue_depth = stats.queue_depth,
enqueue_age_ms = duration_ms(enqueue_age),
"📸 upstream camera queue dropped the oldest frame because it was full"
);
} }
telemetry.record_enqueue( telemetry.record_enqueue(
queue_depth_u32(stats.queue_depth), queue_depth_u32(stats.queue_depth),

View File

@ -70,6 +70,13 @@ async fn run_sync_probe(config: ProbeConfig) -> Result<()> {
let outbound = async_stream::stream! { let outbound = async_stream::stream! {
loop { loop {
let next = video_queue.pop_fresh().await; let next = video_queue.pop_fresh().await;
if next.dropped_stale > 0 {
tracing::warn!(
dropped_stale = next.dropped_stale,
queue_depth = next.queue_depth,
"🧪 sync probe video queue dropped stale packets"
);
}
if let Some(packet) = next.packet { if let Some(packet) = next.packet {
yield packet; yield packet;
continue; continue;
@ -93,6 +100,13 @@ async fn run_sync_probe(config: ProbeConfig) -> Result<()> {
let outbound = async_stream::stream! { let outbound = async_stream::stream! {
loop { loop {
let next = audio_queue.pop_fresh().await; let next = audio_queue.pop_fresh().await;
if next.dropped_stale > 0 {
tracing::warn!(
dropped_stale = next.dropped_stale,
queue_depth = next.queue_depth,
"🧪 sync probe audio queue dropped stale packets"
);
}
if let Some(packet) = next.packet { if let Some(packet) = next.packet {
sent_packets = sent_packets.saturating_add(1); sent_packets = sent_packets.saturating_add(1);
if sent_packets <= 5 || sent_packets.is_multiple_of(500) { if sent_packets <= 5 || sent_packets.is_multiple_of(500) {

View File

@ -1,6 +1,6 @@
[package] [package]
name = "lesavka_common" name = "lesavka_common"
version = "0.14.17" version = "0.14.18"
edition = "2024" edition = "2024"
build = "build.rs" build = "build.rs"

View File

@ -22,6 +22,7 @@ VIDEO_SIZE=${VIDEO_SIZE:-auto}
VIDEO_FPS=${VIDEO_FPS:-30} VIDEO_FPS=${VIDEO_FPS:-30}
VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg} VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-ffmpeg}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto} REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
@ -73,6 +74,7 @@ ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${VIDEO_FPS}" \ "${VIDEO_FPS}" \
"${VIDEO_FORMAT}" \ "${VIDEO_FORMAT}" \
"${REMOTE_CAPTURE_STACK}" \ "${REMOTE_CAPTURE_STACK}" \
"${REMOTE_PULSE_CAPTURE_TOOL}" \
"${REMOTE_PULSE_VIDEO_MODE}" \ "${REMOTE_PULSE_VIDEO_MODE}" \
"${REMOTE_AUDIO_SOURCE}" \ "${REMOTE_AUDIO_SOURCE}" \
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \ "${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
@ -85,9 +87,10 @@ video_size=$3
video_fps=$4 video_fps=$4
video_format=$5 video_format=$5
remote_capture_stack=$6 remote_capture_stack=$6
remote_pulse_video_mode=$7 remote_pulse_capture_tool=$7
remote_audio_source=$8 remote_pulse_video_mode=$8
remote_audio_quiesce_user_audio=$9 remote_audio_source=$9
remote_audio_quiesce_user_audio=${10}
rm -f "${remote_capture}" rm -f "${remote_capture}"
@ -291,36 +294,76 @@ if [[ "${capture_mode}" == "pwpipe" ]]; then
"${remote_capture}" "${remote_capture}"
elif [[ "${capture_mode}" == "pulse" ]]; then elif [[ "${capture_mode}" == "pulse" ]]; then
printf 'using Pulse source: %s\n' "${pulse_source}" >&2 printf 'using Pulse source: %s\n' "${pulse_source}" >&2
case "${remote_pulse_video_mode}" in case "${remote_pulse_capture_tool}" in
copy) ffmpeg)
ffmpeg -hide_banner -loglevel error -y \ case "${remote_pulse_video_mode}" in
-thread_queue_size 1024 \ copy)
"${video_args[@]}" \ ffmpeg -hide_banner -loglevel error -y \
-i /dev/video0 \ -thread_queue_size 1024 \
-thread_queue_size 1024 \ "${video_args[@]}" \
-f pulse \ -i /dev/video0 \
-i "${pulse_source}" \ -thread_queue_size 1024 \
-t "${capture_seconds}" \ -f pulse \
-c:v copy \ -i "${pulse_source}" \
-c:a pcm_s16le \ -t "${capture_seconds}" \
"${remote_capture}" -c:v copy \
-c:a pcm_s16le \
"${remote_capture}"
;;
cfr)
ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i /dev/video0 \
-thread_queue_size 1024 \
-f pulse \
-i "${pulse_source}" \
-t "${capture_seconds}" \
-vf "fps=${video_fps}" \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
-c:a pcm_s16le \
"${remote_capture}"
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;; ;;
cfr) gst)
ffmpeg -hide_banner -loglevel error -y \ case "${remote_pulse_video_mode}" in
-thread_queue_size 1024 \ copy)
"${video_args[@]}" \ timeout --signal=INT "$((capture_seconds + 3))" \
-i /dev/video0 \ gst-launch-1.0 -q -e \
-thread_queue_size 1024 \ matroskamux name=mux ! filesink location="${remote_capture}" \
-f pulse \ v4l2src device=/dev/video0 do-timestamp=true ! \
-i "${pulse_source}" \ image/jpeg,width="${resolved_video_size%x*}",height="${resolved_video_size#*x}",framerate="${video_fps}"/1 ! \
-t "${capture_seconds}" \ queue ! mux. \
-vf "fps=${video_fps}" \ pulsesrc device="${pulse_source}" do-timestamp=true ! \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \ audio/x-raw,rate=48000,channels=2 ! \
-c:a pcm_s16le \ audioconvert ! audioresample ! queue ! mux. || true
"${remote_capture}" ;;
cfr)
timeout --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device=/dev/video0 do-timestamp=true ! \
image/jpeg,width="${resolved_video_size%x*}",height="${resolved_video_size#*x}",framerate="${video_fps}"/1 ! \
jpegdec ! videoconvert ! videorate ! video/x-raw,framerate="${video_fps}"/1 ! \
jpegenc quality=90 ! image/jpeg,framerate="${video_fps}"/1 ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! queue ! mux. || true
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;; ;;
*) *)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 printf 'unsupported REMOTE_PULSE_CAPTURE_TOOL=%s\n' "${remote_pulse_capture_tool}" >&2
exit 64 exit 64
;; ;;
esac esac

View File

@ -10,7 +10,7 @@ bench = false
[package] [package]
name = "lesavka_server" name = "lesavka_server"
version = "0.14.17" version = "0.14.18"
edition = "2024" edition = "2024"
autobins = false autobins = false