diff --git a/Cargo.lock b/Cargo.lock index f428834..4586813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.22.7" +version = "0.22.8" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.22.7" +version = "0.22.8" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.22.7" +version = "0.22.8" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index e7994b2..9eda67b 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.22.7" +version = "0.22.8" edition = "2024" [dependencies] diff --git a/client/src/app/uplink_media/bundled_media_queue.rs b/client/src/app/uplink_media/bundled_media_queue.rs index ba8987d..1c00177 100644 --- a/client/src/app/uplink_media/bundled_media_queue.rs +++ b/client/src/app/uplink_media/bundled_media_queue.rs @@ -30,6 +30,7 @@ const BUNDLED_AUDIO_MAX_PENDING: usize = 8; #[cfg(not(coverage))] const BUNDLED_VIDEO_AUDIO_GRACE: Duration = Duration::from_millis(30); +const DEFAULT_BUNDLED_AUDIO_VIDEO_MAX_SPAN_MS: u64 = 90; #[cfg(not(coverage))] const BUNDLED_CAPTURE_EVENT_CHANNEL_CAPACITY: usize = 64; @@ -202,3 +203,12 @@ fn bundle_captured_media( } queue.close(); } + +fn bundled_audio_video_max_span() -> Duration { + std::env::var("LESAVKA_BUNDLED_AUDIO_VIDEO_MAX_SPAN_MS") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .map(Duration::from_millis) + .unwrap_or_else(|| Duration::from_millis(DEFAULT_BUNDLED_AUDIO_VIDEO_MAX_SPAN_MS)) +} diff --git a/client/src/app/uplink_media/tests/mod.rs b/client/src/app/uplink_media/tests/mod.rs index 7cfdfea..9d776ec 100644 --- a/client/src/app/uplink_media/tests/mod.rs +++ b/client/src/app/uplink_media/tests/mod.rs @@ -285,3 +285,73 @@ use super::*; && packet.client_queue_depth == popped.queue_depth as u32 })); } + + /// Verifies stale mic packets cannot poison an otherwise fresh video bundle. + /// + /// Inputs: one fresh HEVC frame plus one mic packet older than the live + /// pairing window. Output: the queued bundle keeps video and drops the + /// stale audio. Why: production logs showed Opus-era mic lag creating + /// 250ms+ capture spans that made the server discard whole A/V bundles. + #[cfg(not(coverage))] + #[tokio::test] + async fn stale_audio_is_dropped_before_it_can_poison_video_bundle() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let telemetry_path = temp_dir.path().join("uplink.json"); + let telemetry = + crate::uplink_telemetry::UplinkTelemetryPublisher::new(telemetry_path, true, true); + let camera_telemetry = + telemetry.handle(crate::uplink_telemetry::UpstreamStreamKind::Camera); + let microphone_telemetry = + telemetry.handle(crate::uplink_telemetry::UpstreamStreamKind::Microphone); + let queue: crate::uplink_fresh_queue::FreshPacketQueue = + crate::uplink_fresh_queue::FreshPacketQueue::new(BUNDLED_MEDIA_UPLINK_QUEUE); + let drop_log = std::sync::Arc::new(std::sync::Mutex::new(UplinkDropLogLimiter::new( + "test-bundled-hevc", + "test", + ))); + let mut bundle_seq = 0_u64; + let video = VideoPacket { + pts: 1_000_000, + data: vec![0, 0, 0, 1, 0x26, 0x01, 0xaa], + seq: 10, + client_capture_pts_us: 1_000_000, + client_send_pts_us: 1_005_000, + ..Default::default() + }; + let audio = vec![ + AudioPacket { + pts: 700_000, + data: vec![0x11; 1_920], + seq: 20, + client_capture_pts_us: 700_000, + client_send_pts_us: 1_005_000, + ..Default::default() + }, + AudioPacket { + pts: 1_020_000, + data: vec![0x22; 1_920], + seq: 21, + client_capture_pts_us: 1_020_000, + client_send_pts_us: 1_020_000, + ..Default::default() + }, + ]; + + emit_bundled_media( + 42, + &mut bundle_seq, + Some(video), + audio, + &queue, + &camera_telemetry, + µphone_telemetry, + &drop_log, + ); + + let bundle = queue.pop_fresh().await.packet.expect("queued bundle"); + assert!(bundle.video.is_some()); + assert_eq!(bundle.audio.len(), 1); + assert_eq!(bundle.audio[0].client_capture_pts_us, 1_020_000); + assert_eq!(bundle.capture_start_us, 1_000_000); + assert_eq!(bundle.capture_end_us, 1_020_000); + } diff --git a/client/src/app/uplink_media/uplink_queue_metadata.rs b/client/src/app/uplink_media/uplink_queue_metadata.rs index 15052ee..8151926 100644 --- a/client/src/app/uplink_media/uplink_queue_metadata.rs +++ b/client/src/app/uplink_media/uplink_queue_metadata.rs @@ -18,12 +18,25 @@ fn emit_bundled_media( session_id: u64, bundle_seq: &mut u64, video: Option, - audio: Vec, + mut audio: Vec, queue: &crate::uplink_fresh_queue::FreshPacketQueue, camera_telemetry: &crate::uplink_telemetry::UplinkTelemetryHandle, microphone_telemetry: &crate::uplink_telemetry::UplinkTelemetryHandle, drop_log: &Arc>, ) { + if let Some(video) = video.as_ref() { + let dropped = retain_audio_near_video(video, &mut audio); + if dropped > 0 { + microphone_telemetry.record_stale_drop(dropped as u64); + log_uplink_drop( + drop_log, + UplinkDropReason::Stale, + dropped as u64, + audio.len(), + bundled_audio_video_max_span().as_secs_f32() * 1_000.0, + ); + } + } if video.is_none() && audio.is_empty() { return; } @@ -75,6 +88,27 @@ fn emit_bundled_media( } } +#[cfg(not(coverage))] +/// Drop microphone packets too far away from the video capture timestamp. +/// +/// Inputs: one video packet and the pending audio window. Output: the number of +/// removed audio packets. Why: Opus/GStreamer can occasionally emit delayed +/// mic buffers; bundling those with fresh webcam frames makes the server drop +/// the whole A/V bundle, which hurts video much more than omitting stale mic +/// audio for that frame. +fn retain_audio_near_video(video: &VideoPacket, audio: &mut Vec) -> usize { + if audio.is_empty() { + return 0; + } + let max_span_us = bundled_audio_video_max_span() + .as_micros() + .min(u128::from(u64::MAX)) as u64; + let video_pts = packet_video_capture_pts_us(video); + let before = audio.len(); + audio.retain(|packet| packet_audio_capture_pts_us(packet).abs_diff(video_pts) <= max_span_us); + before.saturating_sub(audio.len()) +} + #[cfg(not(coverage))] /// Keeps `bundled_capture_bounds` explicit because it sits on the live uplink path, where stale media must be dropped instead of queued into latency. /// Inputs are the typed parameters; output is the return value or side effect. diff --git a/client/src/input/camera.rs b/client/src/input/camera.rs index 7fe1a25..5afe6b8 100644 --- a/client/src/input/camera.rs +++ b/client/src/input/camera.rs @@ -5,7 +5,7 @@ use gstreamer as gst; use gstreamer_app as gst_app; use lesavka_common::lesavka::VideoPacket; use std::{ - io::Write, + io::{Read, Write}, os::fd::IntoRawFd, path::{Path, PathBuf}, process::{Child, Command, Stdio}, diff --git a/client/src/input/camera/capture_pipeline.rs b/client/src/input/camera/capture_pipeline.rs index 6cebfcc..69fde96 100644 --- a/client/src/input/camera/capture_pipeline.rs +++ b/client/src/input/camera/capture_pipeline.rs @@ -80,7 +80,7 @@ impl CameraCapture { height, fps, keyframe_interval, - camera_preview_tap_path().is_some(), + camera_preview_tap_path(), ); } let use_mjpg_source = source_profile == CameraSourceProfile::Mjpeg; @@ -308,23 +308,22 @@ impl CameraCapture { height: u32, fps: u32, keyframe_interval: u32, - preview_tap_enabled: bool, + preview_tap_path: Option, ) -> anyhow::Result { - if preview_tap_enabled { - tracing::warn!( - "📸 HEVC NVENC route is active; launcher preview tap is temporarily disabled for this hardware encode path" - ); - } - let bitrate_kbit = env_u32("LESAVKA_CAM_HEVC_KBIT", 3000).max(250); let fps = fps.max(1); let capture_fps = capture_fps.max(1); let keyframe_interval = keyframe_interval.max(1); + let preview_tap_enabled = preview_tap_path.is_some(); let mut command = Command::new("ffmpeg"); command .arg("-hide_banner") .arg("-loglevel") - .arg(std::env::var("LESAVKA_FFMPEG_LOGLEVEL").unwrap_or_else(|_| "warning".into())) + .arg(if preview_tap_enabled { + "quiet".to_string() + } else { + std::env::var("LESAVKA_FFMPEG_LOGLEVEL").unwrap_or_else(|_| "warning".into()) + }) .arg("-nostdin") .arg("-fflags") .arg("nobuffer") @@ -359,13 +358,24 @@ impl CameraCapture { let video_filter = format!("scale={width}:{height}:flags=fast_bilinear,fps={fps},format=nv12"); + let preview_filter = format!( + "[0:v]scale={width}:{height}:flags=fast_bilinear,fps={fps},split=2[vencsrc][vprevsrc];\ + [vencsrc]format=nv12[vencout];[vprevsrc]format=rgba[vprevout]" + ); let bitrate = format!("{bitrate_kbit}k"); - command - .arg("-an") + command.arg("-an") .arg("-sn") - .arg("-dn") - .arg("-vf") - .arg(video_filter) + .arg("-dn"); + if preview_tap_enabled { + command + .arg("-filter_complex") + .arg(preview_filter) + .arg("-map") + .arg("[vencout]"); + } else { + command.arg("-vf").arg(video_filter); + } + command .arg("-c:v") .arg("hevc_nvenc") .arg("-preset") @@ -388,9 +398,23 @@ impl CameraCapture { .arg("1") .arg("-f") .arg("hevc") - .arg("pipe:1") - .stdout(Stdio::piped()) - .stderr(Stdio::null()); + .arg("pipe:1"); + if preview_tap_enabled { + command + .arg("-map") + .arg("[vprevout]") + .arg("-c:v") + .arg("rawvideo") + .arg("-pix_fmt") + .arg("rgba") + .arg("-f") + .arg("rawvideo") + .arg("pipe:2") + .stderr(Stdio::piped()); + } else { + command.stderr(Stdio::null()); + } + command.stdout(Stdio::piped()); tracing::info!( device = dev_label, @@ -402,10 +426,20 @@ impl CameraCapture { output_fps = fps, bitrate_kbit, keyframe_interval, + preview_tap = preview_tap_enabled, "📸 using FFmpeg hevc_nvenc hardware encoder" ); let mut child = command.spawn().context("starting FFmpeg hevc_nvenc camera encoder")?; + let preview_tap_running = if let Some(path) = preview_tap_path { + let stderr = child + .stderr + .take() + .context("FFmpeg hevc_nvenc preview stream was not piped")?; + Some(spawn_ffmpeg_raw_preview_tap(stderr, path, width, height)) + } else { + None + }; let stdout = child .stdout .take() @@ -442,7 +476,7 @@ impl CameraCapture { pipeline, sink, ffmpeg_child: Some(child), - preview_tap_running: None, + preview_tap_running, pts_rebaser: crate::live_capture_clock::DurationPacedSourcePtsRebaser::default(), frame_duration_us: (1_000_000u64 / u64::from(fps.max(1))).max(1), }) diff --git a/client/src/input/camera/preview_tap.rs b/client/src/input/camera/preview_tap.rs index 1cd8537..99a2b6e 100644 --- a/client/src/input/camera/preview_tap.rs +++ b/client/src/input/camera/preview_tap.rs @@ -29,6 +29,77 @@ fn spawn_camera_preview_tap(sink: gst_app::AppSink, path: PathBuf) -> Arc( + mut reader: R, + path: PathBuf, + width: u32, + height: u32, +) -> Arc +where + R: Read + Send + 'static, +{ + let running = Arc::new(AtomicBool::new(true)); + let thread_running = Arc::clone(&running); + thread::spawn(move || { + let width_i32 = i32::try_from(width).unwrap_or(i32::MAX).max(1); + let height_i32 = i32::try_from(height).unwrap_or(i32::MAX).max(1); + let stride = usize::try_from(width) + .ok() + .and_then(|width| width.checked_mul(4)) + .unwrap_or(4); + let frame_len = stride + .checked_mul(usize::try_from(height).unwrap_or(1).max(1)) + .unwrap_or(stride); + let mut frame = vec![0_u8; frame_len]; + let mut wrote_first = false; + while thread_running.load(Ordering::Acquire) { + match reader.read_exact(&mut frame) { + Ok(()) => match write_camera_preview_rgba( + &path, width_i32, height_i32, stride, &frame, + ) { + Ok(info) => { + if !wrote_first { + wrote_first = true; + log_camera_preview_tap_started(&path, &info); + } + } + Err(err) => { + tracing::debug!("📸 FFmpeg preview tap write failed: {err:#}"); + thread::sleep(Duration::from_millis(100)); + } + }, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => { + tracing::debug!("📸 FFmpeg preview tap stopped: {err:#}"); + break; + } + } + } + }); + running +} + +#[cfg(coverage)] +fn spawn_ffmpeg_raw_preview_tap( + _reader: R, + _path: PathBuf, + _width: u32, + _height: u32, +) -> Arc +where + R: Read + Send + 'static, +{ + Arc::new(AtomicBool::new(false)) +} + #[cfg(not(coverage))] fn log_camera_preview_tap_started(path: &Path, info: &CameraPreviewTapInfo) { tracing::info!( @@ -104,11 +175,21 @@ fn write_camera_preview_tap( .filter(|height| *height > 0) .unwrap_or(1); let stride = map.as_slice().len() / row_count; + write_camera_preview_rgba(path, width, height, stride, map.as_slice()) +} + +fn write_camera_preview_rgba( + path: &Path, + width: i32, + height: i32, + stride: usize, + rgba: &[u8], +) -> anyhow::Result { let tmp_path = path.with_extension("tmp"); let mut file = std::fs::File::create(&tmp_path) .with_context(|| format!("creating {}", tmp_path.display()))?; writeln!(file, "LESAVKA_RGBA {width} {height} {stride}")?; - file.write_all(map.as_slice())?; + file.write_all(rgba)?; file.sync_all().ok(); std::fs::rename(&tmp_path, path).with_context(|| format!("publishing {}", path.display()))?; Ok(CameraPreviewTapInfo { diff --git a/client/src/launcher/tests/ui_runtime.rs b/client/src/launcher/tests/ui_runtime.rs index 015d26e..46b9454 100644 --- a/client/src/launcher/tests/ui_runtime.rs +++ b/client/src/launcher/tests/ui_runtime.rs @@ -600,6 +600,21 @@ fn server_chip_state_tracks_connection_not_just_reachability() { assert_eq!(server_version_label(&state), "???"); } +#[test] +fn hid_chip_uses_live_relay_when_capability_probe_is_late() { + let mut state = LauncherState::new(); + state.set_server_available(true); + + assert_eq!( + recovery_usb_health(&state, false), + (StatusLightState::Caution, "Unknown".to_string()) + ); + assert_eq!( + recovery_usb_health(&state, true), + (StatusLightState::Live, "Connected".to_string()) + ); +} + #[test] fn uac_chip_uses_live_microphone_flow_not_only_server_caps() { let mut state = LauncherState::new(); @@ -630,6 +645,16 @@ fn uac_chip_uses_live_microphone_flow_not_only_server_caps() { (StatusLightState::Live, "Opus".to_string()) ); + state.set_server_media_caps(None, None, None, None); + assert_eq!( + recovery_uac_health(&state, false, Some(&healthy)), + (StatusLightState::Caution, "Unknown".to_string()) + ); + assert_eq!( + recovery_uac_health(&state, true, Some(&healthy)), + (StatusLightState::Live, "Opus".to_string()) + ); + state.select_upstream_audio_transport(UpstreamAudioTransport::Pcm); assert_eq!( recovery_uac_health(&state, true, Some(&healthy)), @@ -678,6 +703,16 @@ fn uvc_chip_degrades_when_live_camera_frames_are_not_flowing() { (StatusLightState::Live, "HEVC".to_string()) ); + state.set_server_media_caps(None, None, None, None); + assert_eq!( + recovery_uvc_health(&state, false, Some(&healthy)), + (StatusLightState::Caution, "Unknown".to_string()) + ); + assert_eq!( + recovery_uvc_health(&state, true, Some(&healthy)), + (StatusLightState::Live, "HEVC".to_string()) + ); + state.select_webcam_transport(WebcamTransport::Mjpeg); assert_eq!( recovery_uvc_health(&state, true, Some(&healthy)), diff --git a/client/src/launcher/ui_runtime/status_details.rs b/client/src/launcher/ui_runtime/status_details.rs index 74f042f..125c06a 100644 --- a/client/src/launcher/ui_runtime/status_details.rs +++ b/client/src/launcher/ui_runtime/status_details.rs @@ -119,7 +119,7 @@ fn server_version_label(state: &LauncherState) -> String { } /// Summarize whether the composite USB gadget appears reachable to the host. -fn recovery_usb_health(state: &LauncherState) -> (StatusLightState, String) { +fn recovery_usb_health(state: &LauncherState, relay_live: bool) -> (StatusLightState, String) { if !state.server_available { return (StatusLightState::Idle, "Offline".to_string()); } @@ -130,6 +130,9 @@ fn recovery_usb_health(state: &LauncherState) -> (StatusLightState, String) { return (StatusLightState::Warning, output.to_ascii_uppercase()); } if state.server_camera.is_none() && state.server_microphone.is_none() { + if relay_live { + return (StatusLightState::Live, "Connected".to_string()); + } return (StatusLightState::Caution, "Unknown".to_string()); } if state.server_camera == Some(false) && state.server_microphone == Some(false) { @@ -147,13 +150,23 @@ fn recovery_uac_health( if !state.server_available { return (StatusLightState::Idle, "Offline".to_string()); } + let codec = state.upstream_audio_transport.label().to_string(); if state.server_microphone == Some(false) { return (StatusLightState::Warning, "Missing".to_string()); } if state.server_microphone.is_none() { + if relay_live { + if !state.channels.microphone { + return (StatusLightState::Idle, "Paused".to_string()); + } + let (health, label) = media_stream_health(stream, MediaStreamKind::Microphone); + if matches!(health, StatusLightState::Live) { + return (health, codec); + } + return (health, label); + } return (StatusLightState::Caution, "Unknown".to_string()); } - let codec = state.upstream_audio_transport.label().to_string(); if !relay_live { return (StatusLightState::Live, codec); } @@ -182,6 +195,16 @@ fn recovery_uvc_health( return (StatusLightState::Warning, "Missing".to_string()); } if state.server_camera.is_none() { + if relay_live { + if !state.channels.camera { + return (StatusLightState::Idle, "Paused".to_string()); + } + let (health, label) = media_stream_health(stream, MediaStreamKind::Camera); + if matches!(health, StatusLightState::Live) { + return (health, codec); + } + return (health, label); + } return (StatusLightState::Caution, "Unknown".to_string()); } if !matches!(state.server_camera_output.as_deref(), Some("uvc")) { diff --git a/client/src/launcher/ui_runtime/status_refresh.rs b/client/src/launcher/ui_runtime/status_refresh.rs index b7e612c..4fa42e5 100644 --- a/client/src/launcher/ui_runtime/status_refresh.rs +++ b/client/src/launcher/ui_runtime/status_refresh.rs @@ -85,7 +85,7 @@ pub fn refresh_launcher_ui(widgets: &LauncherWidgets, state: &LauncherState, chi .shortcut_value .set_text(&toggle_key_label(&state.swap_key)); - let (usb_state, usb_value) = recovery_usb_health(state); + let (usb_state, usb_value) = recovery_usb_health(state, relay_live); set_status_light(&widgets.summary.usb_light, usb_state); widgets.summary.usb_value.set_text(&usb_value); widgets.summary.usb_value.set_tooltip_text(Some(&usb_value)); diff --git a/client/src/output/video/monitor_window.rs b/client/src/output/video/monitor_window.rs index e8e1f67..4dbe744 100644 --- a/client/src/output/video/monitor_window.rs +++ b/client/src/output/video/monitor_window.rs @@ -11,9 +11,14 @@ use std::process::Command; use tracing::{debug, error, info, warn}; const SOFTWARE_VIDEO_FALLBACK_ENV: &str = "LESAVKA_ALLOW_SOFTWARE_VIDEO"; +const ALLOW_VULKAN_H264_DECODER_ENV: &str = "LESAVKA_ALLOW_VULKAN_H264_DECODER"; fn software_video_fallback_allowed() -> bool { - std::env::var(SOFTWARE_VIDEO_FALLBACK_ENV) + env_flag_enabled(SOFTWARE_VIDEO_FALLBACK_ENV) +} + +fn env_flag_enabled(name: &str) -> bool { + std::env::var(name) .ok() .is_some_and(|value| { let trimmed = value.trim(); @@ -25,6 +30,14 @@ fn software_video_fallback_allowed() -> bool { }) } +fn stability_software_decode_allowed() -> bool { + software_video_fallback_allowed() +} + +fn vulkan_h264_decoder_allowed() -> bool { + env_flag_enabled(ALLOW_VULKAN_H264_DECODER_ENV) +} + fn is_hardware_h264_decoder(name: &str) -> bool { matches!( name, @@ -76,7 +89,7 @@ fn pick_h264_decoder() -> anyhow::Result { } anyhow::bail!( - "hardware H.264 decoder required, but no buildable NVIDIA/Vulkan/VAAPI/V4L2 decoder was found" + "hardware H.264 decoder required, but no buildable NVIDIA/VAAPI/V4L2 decoder was found; install gst-plugin-va for the libva-nvidia NVDEC route, or set LESAVKA_ALLOW_VULKAN_H264_DECODER=1 only for Vulkan diagnostics" ) } @@ -86,18 +99,19 @@ fn pick_h264_decoder() -> anyhow::Result { /// element names. Why: include-based tests need to protect the same hardware /// route order as the launcher preview path. fn h264_decoder_preference_order() -> Vec<&'static str> { - const HARDWARE: &[&str] = &[ + const PRIMARY_HARDWARE: &[&str] = &[ "nvh264dec", "nvh264sldec", - "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", "v4l2slh264dec", ]; + const VULKAN_HARDWARE: &[&str] = &["vulkanh264dec"]; const SOFTWARE: &[&str] = &["avdec_h264", "openh264dec"]; - let prefer_software = software_video_fallback_allowed() + let auto_software_allowed = stability_software_decode_allowed(); + let prefer_software = auto_software_allowed && std::env::var("LESAVKA_H264_DECODER_PREFERENCE") .ok() .map(|value| { @@ -108,13 +122,20 @@ fn h264_decoder_preference_order() -> Vec<&'static str> { }) .unwrap_or(false); - let mut candidates = Vec::with_capacity(HARDWARE.len() + SOFTWARE.len()); + let mut candidates = + Vec::with_capacity(PRIMARY_HARDWARE.len() + SOFTWARE.len() + VULKAN_HARDWARE.len()); if prefer_software { candidates.extend_from_slice(SOFTWARE); - candidates.extend_from_slice(HARDWARE); + candidates.extend_from_slice(PRIMARY_HARDWARE); + if vulkan_h264_decoder_allowed() { + candidates.extend_from_slice(VULKAN_HARDWARE); + } } else { - candidates.extend_from_slice(HARDWARE); - if software_video_fallback_allowed() { + candidates.extend_from_slice(PRIMARY_HARDWARE); + if vulkan_h264_decoder_allowed() { + candidates.extend_from_slice(VULKAN_HARDWARE); + } + if auto_software_allowed { candidates.extend_from_slice(SOFTWARE); } } diff --git a/client/src/video_support.rs b/client/src/video_support.rs index 69bd08a..0b6273a 100644 --- a/client/src/video_support.rs +++ b/client/src/video_support.rs @@ -3,6 +3,7 @@ use gstreamer as gst; pub const SOFTWARE_VIDEO_FALLBACK_ENV: &str = "LESAVKA_ALLOW_SOFTWARE_VIDEO"; +pub const ALLOW_VULKAN_H264_DECODER_ENV: &str = "LESAVKA_ALLOW_VULKAN_H264_DECODER"; /// Return whether software video fallback is explicitly allowed. /// @@ -12,16 +13,26 @@ pub const SOFTWARE_VIDEO_FALLBACK_ENV: &str = "LESAVKA_ALLOW_SOFTWARE_VIDEO"; /// instead of silently shifting downstream video onto the CPU. #[must_use] pub fn software_video_fallback_allowed() -> bool { - std::env::var(SOFTWARE_VIDEO_FALLBACK_ENV) - .ok() - .is_some_and(|value| { - let trimmed = value.trim(); - !(trimmed.is_empty() - || trimmed.eq_ignore_ascii_case("0") - || trimmed.eq_ignore_ascii_case("false") - || trimmed.eq_ignore_ascii_case("no") - || trimmed.eq_ignore_ascii_case("off")) - }) + env_flag_enabled(SOFTWARE_VIDEO_FALLBACK_ENV) +} + +fn env_flag_enabled(name: &str) -> bool { + std::env::var(name).ok().is_some_and(|value| { + let trimmed = value.trim(); + !(trimmed.is_empty() + || trimmed.eq_ignore_ascii_case("0") + || trimmed.eq_ignore_ascii_case("false") + || trimmed.eq_ignore_ascii_case("no") + || trimmed.eq_ignore_ascii_case("off")) + }) +} + +fn stability_software_decode_allowed() -> bool { + software_video_fallback_allowed() +} + +fn vulkan_h264_decoder_allowed() -> bool { + env_flag_enabled(ALLOW_VULKAN_H264_DECODER_ENV) } #[must_use] @@ -45,7 +56,7 @@ pub fn is_hardware_h264_decoder(name: &str) -> bool { /// `LESAVKA_H264_DECODER_PREFERENCE=hardware|software`. /// Outputs: the chosen decoder element name, or `decodebin` as a last-resort /// error when no hardware decoder is present. -/// Why: Lesavka should use GPU decode on NVIDIA/Vulkan/VAAPI/V4L2-capable clients +/// Why: Lesavka should use GPU decode on NVIDIA/VAAPI/V4L2-capable clients /// and should not hide hardware failures behind CPU decode. #[must_use] #[allow(dead_code)] // retained for include-based tests and diagnostics. @@ -89,28 +100,30 @@ pub fn require_h264_decoder() -> Result { } } - Err("hardware H.264 decoder required, but no buildable NVIDIA/Vulkan/VAAPI/V4L2 decoder was found".to_string()) + Err("hardware H.264 decoder required, but no buildable NVIDIA/VAAPI/V4L2 decoder was found; install gst-plugin-va for the libva-nvidia NVDEC route, or set LESAVKA_ALLOW_VULKAN_H264_DECODER=1 only for Vulkan diagnostics".to_string()) } /// Return automatic H.264 decoder candidates in selection order. /// /// Inputs: `LESAVKA_H264_DECODER_PREFERENCE`, if set. Output: ordered decoder /// element names. Why: tests and diagnostics need to prove proprietary -/// NVIDIA, Vulkan, and VAAPI/V4L2 routes stay ahead of explicit lab fallback. +/// NVIDIA and VAAPI/V4L2 routes stay ahead of explicit lab fallback; Vulkan is +/// opt-in because it has been choppy on the NVIDIA desktop path. #[must_use] pub fn h264_decoder_preference_order() -> Vec<&'static str> { - const HARDWARE: &[&str] = &[ + const PRIMARY_HARDWARE: &[&str] = &[ "nvh264dec", "nvh264sldec", - "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", "v4l2slh264dec", ]; + const VULKAN_HARDWARE: &[&str] = &["vulkanh264dec"]; const SOFTWARE: &[&str] = &["avdec_h264", "openh264dec"]; - let prefer_software = software_video_fallback_allowed() + let auto_software_allowed = stability_software_decode_allowed(); + let prefer_software = auto_software_allowed && std::env::var("LESAVKA_H264_DECODER_PREFERENCE") .ok() .map(|value| { @@ -121,13 +134,20 @@ pub fn h264_decoder_preference_order() -> Vec<&'static str> { }) .unwrap_or(false); - let mut candidates = Vec::with_capacity(HARDWARE.len() + SOFTWARE.len()); + let mut candidates = + Vec::with_capacity(PRIMARY_HARDWARE.len() + SOFTWARE.len() + VULKAN_HARDWARE.len()); if prefer_software { candidates.extend_from_slice(SOFTWARE); - candidates.extend_from_slice(HARDWARE); + candidates.extend_from_slice(PRIMARY_HARDWARE); + if vulkan_h264_decoder_allowed() { + candidates.extend_from_slice(VULKAN_HARDWARE); + } } else { - candidates.extend_from_slice(HARDWARE); - if software_video_fallback_allowed() { + candidates.extend_from_slice(PRIMARY_HARDWARE); + if vulkan_h264_decoder_allowed() { + candidates.extend_from_slice(VULKAN_HARDWARE); + } + if auto_software_allowed { candidates.extend_from_slice(SOFTWARE); } } diff --git a/common/Cargo.toml b/common/Cargo.toml index f8a4624..0a372d3 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.22.7" +version = "0.22.8" edition = "2024" build = "build.rs" diff --git a/docs/operational-env.md b/docs/operational-env.md index ffe9160..3078975 100644 --- a/docs/operational-env.md +++ b/docs/operational-env.md @@ -129,7 +129,8 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_GADGET_SYSFS_ROOT` | server hardware/device override | | `LESAVKA_GIT_SHA` | runtime/install/session override | | `LESAVKA_H264_DECODER` | eye preview/video transport override; names an explicit hardware GStreamer decoder such as `nvh264dec`, `vulkanh264dec`, or `v4l2h264dec`; software names are rejected unless `LESAVKA_ALLOW_SOFTWARE_VIDEO=1` | -| `LESAVKA_H264_DECODER_PREFERENCE` | eye preview/video transport override; `hardware`/unset uses NVIDIA, Vulkan, VAAPI, and V4L2 decode only; `software`/`cpu` is honored only with `LESAVKA_ALLOW_SOFTWARE_VIDEO=1` for lab driver comparisons | +| `LESAVKA_H264_DECODER_PREFERENCE` | eye preview/video transport override; `hardware`/unset uses NVIDIA, VAAPI, and V4L2 decode only; `software`/`cpu` is honored only with `LESAVKA_ALLOW_SOFTWARE_VIDEO=1` for lab driver comparisons | +| `LESAVKA_ALLOW_VULKAN_H264_DECODER` | client downstream video diagnostic override; opt into Vulkan H.264 decode when NVIDIA/VAAPI/V4L2 decode is unavailable or under comparison | | `LESAVKA_ALLOW_SOFTWARE_VIDEO` | video acceleration safety override; when truthy, permits software decode/encode fallbacks for lab/debug runs only | | `LESAVKA_HDMI_CONNECTOR` | server hardware/device override | | `LESAVKA_HDMI_DRIVER` | server hardware/device override | @@ -306,7 +307,10 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_USB_RECOVERY_CYCLE_WAIT_MS` | USB recovery timing override | | `LESAVKA_USB_RECOVERY_FINAL_WAIT_MS` | USB recovery timing override | | `LESAVKA_USB_RECOVERY_REBUILD_WAIT_MS` | USB recovery timing override | -| `LESAVKA_UVC_APP_BLOCK` | server hardware/device override | +| `LESAVKA_UVC_APP_LEAKY_TYPE` | server UVC appsrc freshness override; defaults to `downstream` so stale webcam frames are dropped instead of buffered during decode/gadget stalls | +| `LESAVKA_UVC_APP_MAX_BUFFERS` | server UVC appsrc memory guard; defaults to `4` queued encoded frames | +| `LESAVKA_UVC_APP_MAX_BYTES` | server UVC appsrc memory guard; defaults to `4194304` queued bytes | +| `LESAVKA_UVC_APP_MAX_TIME_NS` | server UVC appsrc memory guard; defaults to `200000000` ns of queued media | | `LESAVKA_UVC_BLOCKING` | server hardware/device override | | `LESAVKA_UVC_BULK` | server hardware/device override | | `LESAVKA_UVC_BUFFER_COUNT` | UVC helper freshness override; number of queued gadget output buffers, defaults to `2` for live-call freshness | diff --git a/scripts/install/client.sh b/scripts/install/client.sh index c50e7e9..4239ff5 100755 --- a/scripts/install/client.sh +++ b/scripts/install/client.sh @@ -436,7 +436,7 @@ pacman_install \ git rustup protobuf abseil-cpp gcc clang llvm-libs compiler-rt evtest base-devel libpulse \ "${PIPEWIRE_PACKAGES[@]}" wireplumber \ alsa-utils gst-plugin-pipewire \ - gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-libav \ + gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-libav gst-plugin-va \ ffmpeg wmctrl qt6-tools wl-clipboard xclip xsel desktop-file-utils openssl ensure_yay() { diff --git a/server/Cargo.toml b/server/Cargo.toml index 681cac2..b63cd9f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.22.7" +version = "0.22.8" edition = "2024" autobins = false diff --git a/server/src/video_sinks/webcam_sink.rs b/server/src/video_sinks/webcam_sink.rs index df74184..7730c2d 100644 --- a/server/src/video_sinks/webcam_sink.rs +++ b/server/src/video_sinks/webcam_sink.rs @@ -65,6 +65,55 @@ fn uvc_mjpeg_v4l2sink_io_mode() -> String { } } +fn positive_u64_env(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(default) +} + +fn uvc_appsrc_max_buffers() -> u64 { + positive_u64_env("LESAVKA_UVC_APP_MAX_BUFFERS", 4) +} + +fn uvc_appsrc_max_bytes() -> u64 { + positive_u64_env("LESAVKA_UVC_APP_MAX_BYTES", 4 * 1024 * 1024) +} + +fn uvc_appsrc_max_time_ns() -> u64 { + positive_u64_env("LESAVKA_UVC_APP_MAX_TIME_NS", 200_000_000) +} + +fn uvc_appsrc_leaky_type() -> String { + std::env::var("LESAVKA_UVC_APP_LEAKY_TYPE") + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| matches!(value.as_str(), "downstream" | "upstream" | "none")) + .unwrap_or_else(|| "downstream".to_string()) +} + +/// Bound the UVC ingress queue so decode/UVC stalls cannot turn into RSS growth. +/// +/// Inputs: the UVC `appsrc`. Output: side-effect-only GStreamer properties. +/// Why: live webcam output should prefer dropping stale frames over buffering +/// seconds or minutes of encoded media when the physical sink falls behind. +fn configure_uvc_appsrc(appsrc: &gst_app::AppSrc) { + appsrc.set_property("block", false); + if appsrc.has_property("max-buffers", None) { + appsrc.set_property("max-buffers", uvc_appsrc_max_buffers()); + } + if appsrc.has_property("max-bytes", None) { + appsrc.set_property("max-bytes", uvc_appsrc_max_bytes()); + } + if appsrc.has_property("max-time", None) { + appsrc.set_property("max-time", uvc_appsrc_max_time_ns()); + } + if appsrc.has_property("leaky-type", None) { + appsrc.set_property_from_str("leaky-type", &uvc_appsrc_leaky_type()); + } +} + impl WebcamSink { /// Build a new webcam sink pipeline. /// @@ -85,6 +134,7 @@ impl WebcamSink { src.set_is_live(true); src.set_format(gst::Format::Time); src.set_property("do-timestamp", &false); + configure_uvc_appsrc(&src); let sink = gst::ElementFactory::make("fakesink") .build() @@ -130,11 +180,7 @@ impl WebcamSink { src.set_is_live(true); src.set_format(gst::Format::Time); src.set_property("do-timestamp", false); - let block = std::env::var("LESAVKA_UVC_APP_BLOCK") - .ok() - .map(|value| value != "0") - .unwrap_or(false); - src.set_property("block", block); + configure_uvc_appsrc(&src); if clock_align_enabled { crate::media_timing::prepare_pipeline_clock_sync(&pipeline); } @@ -492,4 +538,49 @@ mod tests { ); }); } + + #[test] + fn uvc_appsrc_limits_default_to_freshness_first_bounds() { + temp_env::with_var_unset("LESAVKA_UVC_APP_MAX_BUFFERS", || { + temp_env::with_var_unset("LESAVKA_UVC_APP_MAX_BYTES", || { + temp_env::with_var_unset("LESAVKA_UVC_APP_MAX_TIME_NS", || { + temp_env::with_var_unset("LESAVKA_UVC_APP_LEAKY_TYPE", || { + assert_eq!(super::uvc_appsrc_max_buffers(), 4); + assert_eq!(super::uvc_appsrc_max_bytes(), 4 * 1024 * 1024); + assert_eq!(super::uvc_appsrc_max_time_ns(), 200_000_000); + assert_eq!(super::uvc_appsrc_leaky_type(), "downstream"); + }); + }); + }); + }); + } + + #[test] + fn uvc_appsrc_limits_accept_positive_safe_overrides_only() { + temp_env::with_var("LESAVKA_UVC_APP_MAX_BUFFERS", Some("6"), || { + temp_env::with_var("LESAVKA_UVC_APP_MAX_BYTES", Some("1048576"), || { + temp_env::with_var("LESAVKA_UVC_APP_MAX_TIME_NS", Some("100000000"), || { + temp_env::with_var("LESAVKA_UVC_APP_LEAKY_TYPE", Some("upstream"), || { + assert_eq!(super::uvc_appsrc_max_buffers(), 6); + assert_eq!(super::uvc_appsrc_max_bytes(), 1_048_576); + assert_eq!(super::uvc_appsrc_max_time_ns(), 100_000_000); + assert_eq!(super::uvc_appsrc_leaky_type(), "upstream"); + }); + }); + }); + }); + + temp_env::with_var("LESAVKA_UVC_APP_MAX_BUFFERS", Some("0"), || { + temp_env::with_var("LESAVKA_UVC_APP_MAX_BYTES", Some("nope"), || { + temp_env::with_var("LESAVKA_UVC_APP_MAX_TIME_NS", Some("0"), || { + temp_env::with_var("LESAVKA_UVC_APP_LEAKY_TYPE", Some("sideways"), || { + assert_eq!(super::uvc_appsrc_max_buffers(), 4); + assert_eq!(super::uvc_appsrc_max_bytes(), 4 * 1024 * 1024); + assert_eq!(super::uvc_appsrc_max_time_ns(), 200_000_000); + assert_eq!(super::uvc_appsrc_leaky_type(), "downstream"); + }); + }); + }); + }); + } } diff --git a/tests/compatibility/client/video_support/client_video_support_include_contract.rs b/tests/compatibility/client/video_support/client_video_support_include_contract.rs index baeed2b..6548282 100644 --- a/tests/compatibility/client/video_support/client_video_support_include_contract.rs +++ b/tests/compatibility/client/video_support/client_video_support_include_contract.rs @@ -72,18 +72,55 @@ fn decoder_override_ignores_blank_or_unknown_values() { #[serial] fn decoder_auto_order_supports_proprietary_and_open_source_routes() { with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { - let order = video_support::h264_decoder_preference_order(); - assert_eq!(order.first(), Some(&"nvh264dec")); - assert!(order.contains(&"nvh264sldec")); - assert!(order.contains(&"vulkanh264dec")); - assert!(order.contains(&"vah264dec")); - assert!(order.contains(&"vaapih264dec")); - assert!(order.contains(&"v4l2h264dec")); - assert!(order.contains(&"v4l2slh264dec")); - assert!( - !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), - "CPU decoders should not be automatic production candidates" - ); + with_var("LESAVKA_ALLOW_VULKAN_H264_DECODER", None::<&str>, || { + with_var("LESAVKA_ALLOW_SOFTWARE_VIDEO", None::<&str>, || { + let order = video_support::h264_decoder_preference_order(); + assert_eq!(order.first(), Some(&"nvh264dec")); + assert!(order.contains(&"nvh264sldec")); + assert!(order.contains(&"vah264dec")); + assert!(order.contains(&"vaapih264dec")); + assert!(order.contains(&"v4l2h264dec")); + assert!(order.contains(&"v4l2slh264dec")); + assert!( + !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), + "software decoders should require explicit lab fallback" + ); + assert!( + !order.contains(&"vulkanh264dec"), + "Vulkan decode should be opt-in because it is choppy on the NVIDIA desktop path" + ); + }); + }); + }); +} + +#[test] +#[serial] +fn vulkan_decoder_is_opt_in_for_driver_diagnostics() { + with_var("LESAVKA_ALLOW_VULKAN_H264_DECODER", Some("1"), || { + with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { + let order = video_support::h264_decoder_preference_order(); + assert_eq!(order.first(), Some(&"nvh264dec")); + assert!(order.contains(&"vulkanh264dec")); + assert!( + !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), + "Vulkan opt-in should still preserve the no-software production rule" + ); + }); + }); +} + +#[test] +#[serial] +fn production_auto_order_keeps_cpu_decoders_out() { + with_var("LESAVKA_ALLOW_SOFTWARE_VIDEO", None::<&str>, || { + with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { + let order = video_support::h264_decoder_preference_order(); + assert!( + !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), + "production auto-selection should not select CPU decoders" + ); + }); }); } diff --git a/tests/contract/client/input/camera/client_camera_include_contract.rs b/tests/contract/client/input/camera/client_camera_include_contract.rs index 3e97e99..b7fe83e 100644 --- a/tests/contract/client/input/camera/client_camera_include_contract.rs +++ b/tests/contract/client/input/camera/client_camera_include_contract.rs @@ -150,6 +150,38 @@ mod camera_include_contract { } } + #[test] + fn ffmpeg_nvenc_route_keeps_launcher_preview_tap_alive() { + let pipeline_source = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/client/src/input/camera/capture_pipeline.rs" + )); + let tap_source = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/client/src/input/camera/preview_tap.rs" + )); + + for expected in [ + "spawn_ffmpeg_raw_preview_tap", + "-filter_complex", + "[vencsrc]format=nv12[vencout];[vprevsrc]format=rgba[vprevout]", + ".arg(\"[vprevout]\")", + ".arg(\"rawvideo\")", + ".arg(\"pipe:2\")", + "preview_tap_running", + ] { + assert!( + pipeline_source.contains(expected) || tap_source.contains(expected), + "FFmpeg/NVENC camera path should preserve preview marker {expected}" + ); + } + + assert!( + !pipeline_source.contains("launcher preview tap is temporarily disabled"), + "NVENC route must not regress to disabling the launcher webcam preview" + ); + } + #[test] #[cfg(coverage)] fn camera_bus_logger_coverage_stub_is_non_blocking() { diff --git a/tests/contract/client/output/video/client_output_video_include_contract.rs b/tests/contract/client/output/video/client_output_video_include_contract.rs index a5a1e88..5e47f63 100644 --- a/tests/contract/client/output/video/client_output_video_include_contract.rs +++ b/tests/contract/client/output/video/client_output_video_include_contract.rs @@ -156,18 +156,39 @@ mod video_include_contract { #[serial] fn h264_decoder_selection_requires_hardware_unless_lab_fallback_is_explicit() { with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { - let order = h264_decoder_preference_order(); - assert_eq!(order.first(), Some(&"nvh264dec")); - assert!(order.contains(&"nvh264sldec")); - assert!(order.contains(&"vulkanh264dec")); - assert!(order.contains(&"vah264dec")); - assert!(order.contains(&"vaapih264dec")); - assert!(order.contains(&"v4l2h264dec")); - assert!(order.contains(&"v4l2slh264dec")); - assert!( - !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), - "software decoders should be absent from production order" - ); + with_var("LESAVKA_ALLOW_SOFTWARE_VIDEO", None::<&str>, || { + with_var("LESAVKA_ALLOW_VULKAN_H264_DECODER", None::<&str>, || { + let order = h264_decoder_preference_order(); + assert_eq!(order.first(), Some(&"nvh264dec")); + assert!(order.contains(&"nvh264sldec")); + assert!(order.contains(&"vah264dec")); + assert!(order.contains(&"vaapih264dec")); + assert!(order.contains(&"v4l2h264dec")); + assert!(order.contains(&"v4l2slh264dec")); + assert!(!order.contains(&"avdec_h264")); + assert!(!order.contains(&"openh264dec")); + assert!(!order.contains(&"vulkanh264dec")); + }); + }); + }); + + with_var("LESAVKA_ALLOW_VULKAN_H264_DECODER", Some("1"), || { + with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { + let order = h264_decoder_preference_order(); + assert!(order.contains(&"vulkanh264dec")); + assert!( + !order.contains(&"avdec_h264") && !order.contains(&"openh264dec"), + "Vulkan diagnostics should not imply software fallback" + ); + }); + }); + + with_var("LESAVKA_ALLOW_SOFTWARE_VIDEO", None::<&str>, || { + with_var("LESAVKA_H264_DECODER_PREFERENCE", None::<&str>, || { + let order = h264_decoder_preference_order(); + assert!(!order.contains(&"avdec_h264")); + assert!(!order.contains(&"openh264dec")); + }); }); with_var("LESAVKA_ALLOW_SOFTWARE_VIDEO", Some("1"), || { diff --git a/tests/installer/scripts/install/client_install_script_contract.rs b/tests/installer/scripts/install/client_install_script_contract.rs index 833dd5b..df55ade 100644 --- a/tests/installer/scripts/install/client_install_script_contract.rs +++ b/tests/installer/scripts/install/client_install_script_contract.rs @@ -97,6 +97,7 @@ fn client_install_reports_nvidia_and_open_source_media_routes() { "vulkanh264enc", "vulkanh264dec", "vulkanh265dec", + "gst-plugin-va", "vah265enc", "vaapih265enc", "v4l2h265enc",