diff --git a/client/src/input/camera.rs b/client/src/input/camera.rs index e696215..7a8b552 100644 --- a/client/src/input/camera.rs +++ b/client/src/input/camera.rs @@ -195,6 +195,20 @@ mod tests { }); } + #[test] + #[serial] + /// Vulkan H.264 hardware encode should stay live-call shaped when available. + fn vulkan_h264_encoder_options_keep_cbr_and_keyframes() { + temp_env::with_var("LESAVKA_CAM_H264_KBIT", Some("6000"), || { + let options = CameraCapture::encoder_options("vulkanh264enc", Some("idr-period"), 30); + + assert_eq!( + options, + "vulkanh264enc bitrate=6000 rate-control=cbr idr-period=30" + ); + }); + } + #[test] #[serial] /// HEVC should recover quickly after freshness drops without changing H.264 knobs. diff --git a/client/src/input/camera/capture_pipeline.rs b/client/src/input/camera/capture_pipeline.rs index 66c20c3..4750ee1 100644 --- a/client/src/input/camera/capture_pipeline.rs +++ b/client/src/input/camera/capture_pipeline.rs @@ -75,7 +75,7 @@ impl CameraCapture { if output_hevc { Self::choose_hevc_encoder() } else { - ("x264enc", Some("key-int-max")) + Self::choose_encoder() } } else if output_hevc { Self::choose_hevc_encoder() @@ -84,7 +84,7 @@ impl CameraCapture { }; match source_profile { CameraSourceProfile::Mjpeg if !output_mjpeg => { - tracing::info!("📸 using MJPG source with software encode"); + tracing::info!("📸 using MJPG source with transcoded output"); } CameraSourceProfile::AutoDecode => { tracing::info!("📸 using auto-decoded webcam source (raw/MJPEG accepted)"); @@ -121,6 +121,12 @@ impl CameraCapture { "videoconvert ! video/x-raw,format=I420,width={width},height={height},framerate={fps}/1 !" ), #[cfg(not(coverage))] + "vulkanh264enc" => + format!( + "videoconvert ! video/x-raw,format=NV12,width={width},height={height},framerate={fps}/1 ! \ + vulkanupload ! video/x-raw(memory:VulkanImage),format=NV12,width={width},height={height},framerate={fps}/1 !" + ), + #[cfg(not(coverage))] "vaapih264enc" | "vah265enc" | "vaapih265enc" | "v4l2h265enc" => format!( "videoconvert ! video/x-raw,format=NV12,width={width},height={height},framerate={fps}/1 !" @@ -140,7 +146,7 @@ impl CameraCapture { // tracing::debug!(%desc, "📸 pipeline-desc"); // Build a pipeline that works for any of the three encoders. // * NVIDIA encoders prefer NV12, using NVMM when Jetson's converter is present. - // * VAAPI/V4L2 hardware encoders also get explicit NV12 system-memory caps. + // * Vulkan/VAAPI/V4L2 hardware encoders also get explicit NV12 caps. // * x264enc/x265enc keep their software-friendly raw caps. let preview_tap_path = camera_preview_tap_path(); let preview_tap_branch = camera_preview_tap_branch(width, height, fps); diff --git a/client/src/input/camera/encoder_selection.rs b/client/src/input/camera/encoder_selection.rs index db4cd41..3a34189 100644 --- a/client/src/input/camera/encoder_selection.rs +++ b/client/src/input/camera/encoder_selection.rs @@ -4,6 +4,7 @@ impl CameraCapture { fn pick_encoder() -> (&'static str, &'static str) { let encoders = &[ ("nvh264enc", "video/x-raw(memory:NVMM),format=NV12"), + ("vulkanh264enc", "video/x-raw(memory:VulkanImage),format=NV12"), ("vaapih264enc", "video/x-raw,format=NV12"), ("v4l2h264enc", "video/x-raw"), // RPi, Jetson, etc. ("x264enc", "video/x-raw"), // software @@ -33,6 +34,12 @@ impl CameraCapture { ), ); } + if buildable_encoder("vulkanh264enc") { + return ( + "vulkanh264enc", + supported_encoder_property("vulkanh264enc", &["idr-period"]), + ); + } if buildable_encoder("vaapih264enc") { return ( "vaapih264enc", @@ -56,6 +63,7 @@ impl CameraCapture { .map(str::trim) { Some("nvh264enc") => ("nvh264enc", None), + Some("vulkanh264enc") => ("vulkanh264enc", Some("idr-period")), Some("vaapih264enc") => ("vaapih264enc", Some("keyframe-period")), Some("v4l2h264enc") => ("v4l2h264enc", Some("idrcount")), _ => ("x264enc", Some("key-int-max")), @@ -107,6 +115,12 @@ impl CameraCapture { format!( "{enc} tune=zerolatency speed-preset=faster bitrate={bitrate_kbit}{keyframe_opt}" ) + } else if enc == "vulkanh264enc" { + let bitrate_kbit = env_u32("LESAVKA_CAM_H264_KBIT", 4500); + let keyframe_opt = kf_prop + .map(|property| format!(" {property}={keyframe_interval}")) + .unwrap_or_default(); + format!("{enc} bitrate={bitrate_kbit} rate-control=cbr{keyframe_opt}") } else if enc == "x265enc" { let bitrate_kbit = env_u32("LESAVKA_CAM_HEVC_KBIT", 3000); let keyframe_opt = kf_prop diff --git a/client/src/launcher/diagnostics/recommendations.rs b/client/src/launcher/diagnostics/recommendations.rs index b195408..eb68021 100644 --- a/client/src/launcher/diagnostics/recommendations.rs +++ b/client/src/launcher/diagnostics/recommendations.rs @@ -269,6 +269,7 @@ fn decoder_label_is_hardware(label: &str) -> bool { let lower = label.to_ascii_lowercase(); lower.contains("nvh264dec") || lower.contains("nvdec") + || lower.contains("vulkanh264dec") || lower.contains("vah264dec") || lower.contains("vaapih264dec") || lower.contains("v4l2slh264dec") diff --git a/client/src/launcher/preview/feed_runtime.rs b/client/src/launcher/preview/feed_runtime.rs index 48dc556..8193161 100644 --- a/client/src/launcher/preview/feed_runtime.rs +++ b/client/src/launcher/preview/feed_runtime.rs @@ -14,7 +14,7 @@ impl PreviewFeed { let session_active_flag = Arc::clone(&session_active); let active_bindings_flag = Arc::clone(&active_bindings); let running_flag = Arc::clone(&running); - std::thread::spawn(move || { + let worker = std::thread::spawn(move || { if let Err(err) = run_preview_feed( server_addr, monitor_id, @@ -46,6 +46,7 @@ impl PreviewFeed { session_active, active_bindings, running, + worker: Arc::new(Mutex::new(Some(worker))), profile, disabled: false, }) @@ -61,6 +62,7 @@ impl PreviewFeed { session_active: Arc::new(AtomicBool::new(false)), active_bindings: Arc::new(AtomicUsize::new(0)), running: Arc::new(AtomicBool::new(false)), + worker: Arc::new(Mutex::new(None)), profile, disabled: true, } @@ -95,6 +97,11 @@ impl PreviewFeed { }, true, ); + if let Ok(mut worker) = self.worker.lock() + && let Some(handle) = worker.take() + { + let _ = handle.join(); + } } fn replace_status(&self, status: impl Into, clear_picture: bool) { @@ -236,7 +243,7 @@ fn run_preview_feed( } }); } - { + let sample_worker = { let shared = Arc::clone(&shared); let appsink = appsink.clone(); let parser = parser.clone(); @@ -268,19 +275,20 @@ fn run_preview_feed( } } } - }); - } + }) + }; let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .context("building preview tokio runtime")?; + let running_for_loop = Arc::clone(&running); let _ = rt.block_on(async move { let mut was_active = false; let mut retry_delay = Duration::from_millis(750); loop { - if !running.load(Ordering::Relaxed) { + if !running_for_loop.load(Ordering::Relaxed) { break; } let active_now = session_active.load(Ordering::Relaxed) @@ -391,7 +399,7 @@ fn run_preview_feed( ); loop { if !session_active.load(Ordering::Relaxed) - || !running.load(Ordering::Relaxed) + || !running_for_loop.load(Ordering::Relaxed) || active_bindings.load(Ordering::Relaxed) == 0 { break; @@ -487,6 +495,8 @@ fn run_preview_feed( }); let _ = pipeline.set_state(gst::State::Null); + running.store(false, Ordering::Relaxed); + let _ = sample_worker.join(); Ok(()) } diff --git a/client/src/launcher/preview/feed_state.rs b/client/src/launcher/preview/feed_state.rs index 11eb398..78c5669 100644 --- a/client/src/launcher/preview/feed_state.rs +++ b/client/src/launcher/preview/feed_state.rs @@ -39,6 +39,7 @@ struct PreviewFeed { session_active: Arc, active_bindings: Arc, running: Arc, + worker: Arc>>>, profile: PreviewProfile, disabled: bool, } diff --git a/client/src/launcher/preview/frame_telemetry.rs b/client/src/launcher/preview/frame_telemetry.rs index 7320394..32b5231 100644 --- a/client/src/launcher/preview/frame_telemetry.rs +++ b/client/src/launcher/preview/frame_telemetry.rs @@ -113,11 +113,15 @@ fn sanitize_preview_request( requested_fps: u32, max_bitrate_kbit: u32, ) -> (i32, i32, u32, u32) { + let requested_width = requested_width.max(2); + let requested_height = requested_height.max(2); + let requested_fps = requested_fps.max(1); + let max_bitrate_kbit = max_bitrate_kbit.max(800); ( - requested_width.max(2), - requested_height.max(2), - requested_fps.max(1), - max_bitrate_kbit.max(800), + requested_width.min(INLINE_PREVIEW_REQUEST_WIDTH), + requested_height.min(INLINE_PREVIEW_REQUEST_HEIGHT), + requested_fps.min(INLINE_PREVIEW_REQUEST_FPS), + max_bitrate_kbit.min(INLINE_PREVIEW_MAX_KBIT), ) } diff --git a/client/src/launcher/preview/preview_core.rs b/client/src/launcher/preview/preview_core.rs index def2cff..142cf15 100644 --- a/client/src/launcher/preview/preview_core.rs +++ b/client/src/launcher/preview/preview_core.rs @@ -1,5 +1,5 @@ #[cfg(not(coverage))] -use crate::video_support::pick_h264_decoder; +use crate::video_support::{h264_decoder_launch_fragment, pick_h264_decoder}; #[cfg(not(coverage))] use anyhow::{Context, Result}; #[cfg(not(coverage))] @@ -34,13 +34,13 @@ const PREVIEW_WIDTH: i32 = 960; #[cfg(not(coverage))] const PREVIEW_HEIGHT: i32 = 540; #[cfg(not(coverage))] -const INLINE_PREVIEW_REQUEST_WIDTH: i32 = DEFAULT_EYE_SOURCE_WIDTH; +const INLINE_PREVIEW_REQUEST_WIDTH: i32 = 1280; #[cfg(not(coverage))] -const INLINE_PREVIEW_REQUEST_HEIGHT: i32 = DEFAULT_EYE_SOURCE_HEIGHT; +const INLINE_PREVIEW_REQUEST_HEIGHT: i32 = 720; #[cfg(not(coverage))] -const INLINE_PREVIEW_REQUEST_FPS: u32 = DEFAULT_EYE_SOURCE_FPS; +const INLINE_PREVIEW_REQUEST_FPS: u32 = 30; #[cfg(not(coverage))] -const INLINE_PREVIEW_MAX_KBIT: u32 = DEFAULT_EYE_SOURCE_MAX_KBIT; +const INLINE_PREVIEW_MAX_KBIT: u32 = 6_000; #[cfg(not(coverage))] const DEFAULT_EYE_SOURCE_WIDTH: i32 = 1920; #[cfg(not(coverage))] diff --git a/client/src/launcher/preview/status_pipeline.rs b/client/src/launcher/preview/status_pipeline.rs index 9ce92fb..05b0896 100644 --- a/client/src/launcher/preview/status_pipeline.rs +++ b/client/src/launcher/preview/status_pipeline.rs @@ -184,13 +184,13 @@ fn build_preview_pipeline( profile.requested_width.max(2) as u32, profile.requested_height.max(2) as u32, ); + let decoder_fragment = h264_decoder_launch_fragment(decoder_name); let desc = format!( "appsrc name=src is-live=true format=time do-timestamp=true block=false ! \ queue max-size-buffers=6 max-size-time=0 max-size-bytes=0 leaky=downstream ! \ - h264parse name=preview_parse disable-passthrough=true ! {} name=decoder ! videoconvert ! \ + h264parse name=preview_parse disable-passthrough=true ! {decoder_fragment} ! videoconvert ! \ video/x-raw,format=RGBA,pixel-aspect-ratio=1/1 ! \ appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true", - decoder_name, ); let pipeline = gst::parse::launch(&desc)? .downcast::() @@ -234,6 +234,7 @@ fn preview_decoder_candidates() -> Vec { for name in [ "avdec_h264", "openh264dec", + "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", diff --git a/client/src/launcher/tests/preview.rs b/client/src/launcher/tests/preview.rs index e2827f2..b218ea0 100644 --- a/client/src/launcher/tests/preview.rs +++ b/client/src/launcher/tests/preview.rs @@ -232,9 +232,9 @@ fn breakout_preview_profile_defaults_to_higher_quality() { } #[test] -fn preview_request_sanitizer_keeps_requested_source_geometry() { +fn preview_request_sanitizer_caps_docked_preview_budget() { let adapted = sanitize_preview_request(1920, 1080, 60, 18_000); - assert_eq!(adapted, (1920, 1080, 60, 18_000)); + assert_eq!(adapted, (1280, 720, 30, 6_000)); } #[test] @@ -333,10 +333,10 @@ fn inline_preview_requests_selected_source_profile_on_wire() { if let Some(request) = requests.lock().unwrap().last().cloned() { assert_eq!(request.id, 1); assert_eq!(request.source_id, Some(1)); - assert_eq!(request.requested_width, 1920); - assert_eq!(request.requested_height, 1080); - assert_eq!(request.requested_fps, 60); - assert_eq!(request.max_bitrate, 18_000); + assert_eq!(request.requested_width, 1280); + assert_eq!(request.requested_height, 720); + assert_eq!(request.requested_fps, 30); + assert_eq!(request.max_bitrate, 6_000); preview.shutdown_all(); return; } @@ -385,10 +385,10 @@ fn inline_preview_requests_honest_source_profile_on_wire() { if let Some(request) = requests.lock().unwrap().last().cloned() { assert_eq!(request.id, 1); assert_eq!(request.source_id, Some(1)); - assert_eq!(request.requested_width, 1920); - assert_eq!(request.requested_height, 1080); - assert_eq!(request.requested_fps, 60); - assert_eq!(request.max_bitrate, 18_000); + assert_eq!(request.requested_width, 1280); + assert_eq!(request.requested_height, 720); + assert_eq!(request.requested_fps, 30); + assert_eq!(request.max_bitrate, 6_000); preview.shutdown_all(); return; } @@ -439,8 +439,8 @@ fn inline_preview_requests_native_720p_source_mode_on_wire() { assert_eq!(request.source_id, Some(1)); assert_eq!(request.requested_width, 1280); assert_eq!(request.requested_height, 720); - assert_eq!(request.requested_fps, 60); - assert_eq!(request.max_bitrate, 12_000); + assert_eq!(request.requested_fps, 30); + assert_eq!(request.max_bitrate, 6_000); preview.shutdown_all(); return; } @@ -491,8 +491,8 @@ fn inline_preview_legacy_low_modes_fall_forward_to_720p_on_wire() { assert_eq!(request.source_id, Some(1)); assert_eq!(request.requested_width, 1280); assert_eq!(request.requested_height, 720); - assert_eq!(request.requested_fps, 60); - assert_eq!(request.max_bitrate, 12_000); + assert_eq!(request.requested_fps, 30); + assert_eq!(request.max_bitrate, 6_000); preview.shutdown_all(); return; } @@ -531,8 +531,10 @@ fn preview_can_request_other_eye_as_a_distinct_stream() { if let Some(request) = requests.lock().unwrap().last().cloned() { assert_eq!(request.id, 0); assert_eq!(request.source_id, Some(1)); - assert_eq!(request.requested_width, 1920); - assert_eq!(request.requested_height, 1080); + assert_eq!(request.requested_width, 1280); + assert_eq!(request.requested_height, 720); + assert_eq!(request.requested_fps, 30); + assert_eq!(request.max_bitrate, 6_000); preview.shutdown_all(); return; } diff --git a/client/src/launcher/tests/ui_preview_profiles.rs b/client/src/launcher/tests/ui_preview_profiles.rs index 84731aa..305399d 100644 --- a/client/src/launcher/tests/ui_preview_profiles.rs +++ b/client/src/launcher/tests/ui_preview_profiles.rs @@ -32,19 +32,19 @@ fn fresh_preview_bootstrap_is_overridden_by_launcher_state_profiles() { let bootstrap = preview.profile_for_test(1, PreviewSurface::Inline).unwrap(); assert_eq!(bootstrap.0, 0); - assert_eq!(bootstrap.3, 1920); - assert_eq!(bootstrap.4, 1080); - assert_eq!(bootstrap.5, 60); - assert_eq!(bootstrap.6, 18_000); + assert_eq!(bootstrap.3, 1280); + assert_eq!(bootstrap.4, 720); + assert_eq!(bootstrap.5, 30); + assert_eq!(bootstrap.6, 6_000); apply_preview_profiles(&preview, &state); let inline = preview.profile_for_test(1, PreviewSurface::Inline).unwrap(); assert_eq!(inline.0, 1); - assert_eq!(inline.3, 1920); - assert_eq!(inline.4, 1080); - assert_eq!(inline.5, 60); - assert_eq!(inline.6, 18_000); + assert_eq!(inline.3, 1280); + assert_eq!(inline.4, 720); + assert_eq!(inline.5, 30); + assert_eq!(inline.6, 6_000); let window = preview.profile_for_test(1, PreviewSurface::Window).unwrap(); assert_eq!(window.0, 1); @@ -57,7 +57,7 @@ fn fresh_preview_bootstrap_is_overridden_by_launcher_state_profiles() { } #[test] -fn source_preview_profile_stays_honest_after_apply() { +fn source_preview_profile_caps_inline_after_apply() { let preview = LauncherPreview::new("http://127.0.0.1:1".to_string()).unwrap(); let mut state = LauncherState::default(); state.set_capture_size_preset(1, CaptureSizePreset::P1080); @@ -66,10 +66,10 @@ fn source_preview_profile_stays_honest_after_apply() { let inline = preview.profile_for_test(1, PreviewSurface::Inline).unwrap(); assert_eq!(inline.0, 1); - assert_eq!(inline.3, 1920); - assert_eq!(inline.4, 1080); - assert_eq!(inline.5, 60); - assert_eq!(inline.6, 18_000); + assert_eq!(inline.3, 1280); + assert_eq!(inline.4, 720); + assert_eq!(inline.5, 30); + assert_eq!(inline.6, 6_000); preview.shutdown_all(); } @@ -105,8 +105,8 @@ fn mirrored_preview_profile_inherits_the_source_eye_mode() { assert_eq!(window.0, 1); assert_eq!(inline.3, 1280); assert_eq!(inline.4, 720); - assert_eq!(inline.5, 60); - assert_eq!(inline.6, 12_000); + assert_eq!(inline.5, 30); + assert_eq!(inline.6, 6_000); assert_eq!(window.3, 1280); assert_eq!(window.4, 720); assert_eq!(window.5, 60); diff --git a/client/src/output/video/monitor_window.rs b/client/src/output/video/monitor_window.rs index b781b69..cfe0bef 100644 --- a/client/src/output/video/monitor_window.rs +++ b/client/src/output/video/monitor_window.rs @@ -15,7 +15,7 @@ use tracing::{debug, error, info, warn}; /// Inputs: optional `LESAVKA_H264_DECODER` override and /// `LESAVKA_H264_DECODER_PREFERENCE=hardware|software`. Output: a decoder /// element name. Why: breakout windows should benefit from NVIDIA proprietary -/// decode when it is present, while keeping VAAPI/V4L2 and CPU routes usable +/// decode when it is present, while keeping Vulkan/VAAPI/V4L2 and CPU routes usable /// for open-source-driver machines and debugging. fn pick_h264_decoder() -> String { if let Ok(raw) = std::env::var("LESAVKA_H264_DECODER") { @@ -46,6 +46,7 @@ fn h264_decoder_preference_order() -> Vec<&'static str> { const HARDWARE: &[&str] = &[ "nvh264dec", "nvh264sldec", + "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", @@ -89,6 +90,26 @@ fn buildable_decoder(name: &str) -> bool { gst::ElementFactory::find(name).is_some() && gst::ElementFactory::make(name).build().is_ok() } +/// Build the decoder stage selected by `pick_h264_decoder`. +/// +/// Vulkan decode is the hardware route exposed by some proprietary NVIDIA +/// installs. It outputs Vulkan memory, so download before handing frames to the +/// existing sink path. +fn h264_decoder_launch_fragment(decoder_name: &str) -> String { + h264_decoder_launch_fragment_named(decoder_name, "decoder") +} + +fn h264_decoder_launch_fragment_named(decoder_name: &str, element_name: &str) -> String { + match decoder_name { + "vulkanh264dec" => concat!( + "vulkanh264dec name={element_name} discard-corrupted-frames=true ", + "automatic-request-sync-points=true ! vulkandownload" + ) + .replace("{element_name}", element_name), + name => format!("{name} name={element_name}"), + } +} + pub struct MonitorWindow { _pipeline: gst::Pipeline, src: gst_app::AppSrc, @@ -218,11 +239,12 @@ impl MonitorWindow { "glimagesink name=sink sync=false" }; + let decoder_fragment = h264_decoder_launch_fragment(&decoder_name); let desc = format!( "appsrc name=src is-live=true format=time do-timestamp=true block=false ! \ queue max-size-buffers=2 max-size-time=0 max-size-bytes=0 leaky=downstream ! \ capsfilter caps=video/x-h264,stream-format=byte-stream,alignment=au ! \ - h264parse disable-passthrough=true ! {decoder_name} name=decoder ! videoconvert ! {sink}" + h264parse disable-passthrough=true ! {decoder_fragment} ! videoconvert ! {sink}" ); let pipeline: gst::Pipeline = gst::parse::launch(&desc)? diff --git a/client/src/output/video/unified_monitor.rs b/client/src/output/video/unified_monitor.rs index 293839c..76aef43 100644 --- a/client/src/output/video/unified_monitor.rs +++ b/client/src/output/video/unified_monitor.rs @@ -73,16 +73,18 @@ impl UnifiedMonitorWindow { "glimagesink name=sink sync=false" }; + let decoder_fragment0 = h264_decoder_launch_fragment_named(&decoder_name, "decoder0"); + let decoder_fragment1 = h264_decoder_launch_fragment_named(&decoder_name, "decoder1"); let desc = format!( "compositor name=mix background=black ! videoconvert ! {sink} \ appsrc name=src0 is-live=true format=time do-timestamp=true block=false ! \ queue max-size-buffers=2 max-size-time=0 max-size-bytes=0 leaky=downstream ! \ capsfilter caps=video/x-h264,stream-format=byte-stream,alignment=au ! \ - h264parse disable-passthrough=true ! {decoder_name} name=decoder0 ! videoconvert ! videoscale ! mix. \ + h264parse disable-passthrough=true ! {decoder_fragment0} ! videoconvert ! videoscale ! mix. \ appsrc name=src1 is-live=true format=time do-timestamp=true block=false ! \ queue max-size-buffers=2 max-size-time=0 max-size-bytes=0 leaky=downstream ! \ capsfilter caps=video/x-h264,stream-format=byte-stream,alignment=au ! \ - h264parse disable-passthrough=true ! {decoder_name} name=decoder1 ! videoconvert ! videoscale ! mix." + h264parse disable-passthrough=true ! {decoder_fragment1} ! videoconvert ! videoscale ! mix." ); let pipeline: gst::Pipeline = gst::parse::launch(&desc)? diff --git a/client/src/sync_probe/capture/video_packets.rs b/client/src/sync_probe/capture/video_packets.rs index b2b9495..64e98b3 100644 --- a/client/src/sync_probe/capture/video_packets.rs +++ b/client/src/sync_probe/capture/video_packets.rs @@ -220,18 +220,26 @@ fn build_encoded_pipeline(camera: CameraConfig) -> Result { /// Why: this probe should run on different developer hosts without hardcoding a /// single hardware encoder, while still preferring low-latency behavior. fn pick_h264_encoder(fps: u32) -> Result { + let fps = fps.max(1); + if gst::ElementFactory::find("vulkanh264enc").is_some() { + return Ok(format!( + "video/x-raw,format=NV12 ! vulkanupload ! \ + video/x-raw(memory:VulkanImage),format=NV12 ! \ + vulkanh264enc bitrate=2500 rate-control=cbr idr-period={fps}" + )); + } + if gst::ElementFactory::find("v4l2h264enc").is_some() { + return Ok("video/x-raw,format=NV12 ! v4l2h264enc".to_string()); + } if gst::ElementFactory::find("x264enc").is_some() { return Ok(format!( "x264enc tune=zerolatency speed-preset=ultrafast bitrate=2500 key-int-max={}", - fps.max(1) + fps )); } if gst::ElementFactory::find("openh264enc").is_some() { return Ok("openh264enc bitrate=2500000".to_string()); } - if gst::ElementFactory::find("v4l2h264enc").is_some() { - return Ok("v4l2h264enc".to_string()); - } bail!("no usable H.264 encoder found for sync probe") } diff --git a/client/src/video_support.rs b/client/src/video_support.rs index 38deda1..c19710c 100644 --- a/client/src/video_support.rs +++ b/client/src/video_support.rs @@ -9,7 +9,7 @@ use gstreamer as gst; /// `LESAVKA_H264_DECODER_PREFERENCE=hardware|software`. /// Outputs: the chosen decoder element name, or `decodebin` as a last-resort /// fallback when no explicit decoder is present. -/// Why: Lesavka should use GPU decode on NVIDIA/VAAPI/V4L2-capable clients +/// Why: Lesavka should use GPU decode on NVIDIA/Vulkan/VAAPI/V4L2-capable clients /// when possible, while keeping an explicit CPU route for open-source driver /// comparisons and driver debugging. #[must_use] @@ -37,12 +37,13 @@ pub fn pick_h264_decoder() -> String { /// /// Inputs: `LESAVKA_H264_DECODER_PREFERENCE`, if set. Output: ordered decoder /// element names. Why: tests and diagnostics need to prove both proprietary -/// NVIDIA and open-source VAAPI/V4L2 routes stay available before CPU fallback. +/// NVIDIA, Vulkan, and VAAPI/V4L2 routes stay available before CPU fallback. #[must_use] pub fn h264_decoder_preference_order() -> Vec<&'static str> { const HARDWARE: &[&str] = &[ "nvh264dec", "nvh264sldec", + "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", @@ -71,6 +72,36 @@ pub fn h264_decoder_preference_order() -> Vec<&'static str> { candidates } +/// Return a parse-launch fragment for the selected H.264 decoder. +/// +/// Inputs: decoder element name. Output: a pipeline fragment with a stable +/// `decoder` element name. Why: Vulkan decoders output GPU memory, so they need +/// an explicit download step before the existing CPU-side sinks can consume +/// frames; keeping that in one helper prevents hardware decode from being +/// selected and then immediately failing link negotiation. +#[must_use] +pub fn h264_decoder_launch_fragment(decoder_name: &str) -> String { + h264_decoder_launch_fragment_named(decoder_name, "decoder") +} + +/// Return a parse-launch fragment for the selected H.264 decoder with a caller-owned element name. +/// +/// Inputs: decoder element name plus the element name to put in the pipeline. +/// Output: a pipeline fragment. Why: unified downstream rendering needs two +/// independent decoder elements, while Vulkan still needs an explicit +/// download-to-system-memory stage after each decoder. +#[must_use] +pub fn h264_decoder_launch_fragment_named(decoder_name: &str, element_name: &str) -> String { + match decoder_name { + "vulkanh264dec" => concat!( + "vulkanh264dec name={element_name} discard-corrupted-frames=true ", + "automatic-request-sync-points=true ! vulkandownload" + ) + .replace("{element_name}", element_name), + name => format!("{name} name={element_name}"), + } +} + fn buildable_decoder(name: &str) -> bool { #[cfg(coverage)] if std::env::var("TEST_FAIL_GST_INIT").is_ok() { diff --git a/scripts/install/client.sh b/scripts/install/client.sh index 3fefa4f..b316753 100755 --- a/scripts/install/client.sh +++ b/scripts/install/client.sh @@ -150,6 +150,7 @@ report_client_media_acceleration() { log "1e. Inspecting client media acceleration routes" local hevc_encoder="" + local h264_encoder="" local h264_decoder="" local opus_encoder="" local opus_decoder="" @@ -163,9 +164,16 @@ report_client_media_acceleration() { vaapih265enc \ v4l2h265enc \ x265enc || true) + h264_encoder=$(first_available_gst_element \ + nvh264enc \ + vulkanh264enc \ + vaapih264enc \ + v4l2h264enc \ + x264enc || true) h264_decoder=$(first_available_gst_element \ nvh264dec \ nvh264sldec \ + vulkanh264dec \ vah264dec \ vaapih264dec \ v4l2h264dec \ @@ -181,6 +189,11 @@ report_client_media_acceleration() { proprietary_bits+=("$element") fi done + for element in vulkanh264enc vulkanh264dec vulkanh265dec; do + if gst_element_available "$element"; then + opensource_bits+=("$element") + fi + done for element in vah265enc vaapih265enc v4l2h265enc vah264dec vaapih264dec v4l2h264dec v4l2slh264dec; do if gst_element_available "$element"; then opensource_bits+=("$element") @@ -189,6 +202,12 @@ report_client_media_acceleration() { if command -v nvidia-smi >/dev/null 2>&1; then echo " ↪ nvidia-smi is available; proprietary NVIDIA driver tooling is present" + if [[ -z $hevc_encoder ]] || [[ $hevc_encoder == x265enc ]]; then + if gst-inspect-1.0 nvcodec 2>&1 | grep -q 'Unable to initialize CUDA library'; then + echo "⚠️ NVIDIA nvcodec is installed but CUDA initialization failed; NVENC HEVC is unavailable to GStreamer." + echo " Vulkan H.264 hardware encode/decode can still be used when the relay profile is H.264." + fi + fi else echo " ↪ nvidia-smi is not available; NVIDIA proprietary tooling was not detected" fi @@ -199,9 +218,9 @@ report_client_media_acceleration() { echo " ↪ proprietary NVIDIA GStreamer route: not exposed" fi if [[ ${#opensource_bits[@]} -gt 0 ]]; then - echo " ↪ open-source VAAPI/V4L2 GStreamer route: ${opensource_bits[*]}" + echo " ↪ Vulkan/VAAPI/V4L2 GStreamer route: ${opensource_bits[*]}" else - echo " ↪ open-source VAAPI/V4L2 GStreamer route: not exposed" + echo " ↪ Vulkan/VAAPI/V4L2 GStreamer route: not exposed" fi if [[ -n $hevc_encoder ]]; then @@ -209,6 +228,11 @@ report_client_media_acceleration() { else echo "⚠️ no HEVC encoder was detected; upstream HEVC will need NVIDIA/VAAPI/V4L2 or x265enc" fi + if [[ -n $h264_encoder ]]; then + echo " ↪ upstream H.264 encoder candidate: $h264_encoder" + else + echo "⚠️ no H.264 encoder was detected; hardware H.264 uplink will need NVIDIA/Vulkan/VAAPI/V4L2 or x264enc" + fi if [[ -n $h264_decoder ]]; then echo " ↪ downstream H.264 decoder candidate: $h264_decoder" else diff --git a/tests/compatibility/client/video_support/client_video_support_include_contract.rs b/tests/compatibility/client/video_support/client_video_support_include_contract.rs index aca0152..53852e8 100644 --- a/tests/compatibility/client/video_support/client_video_support_include_contract.rs +++ b/tests/compatibility/client/video_support/client_video_support_include_contract.rs @@ -45,6 +45,7 @@ fn decoder_auto_order_supports_proprietary_and_open_source_routes() { let order = video_support::h264_decoder_preference_order(); assert_eq!(order.first(), Some(&"nvh264dec")); assert!(order.contains(&"nvh264sldec")); + assert!(order.contains(&"vulkanh264dec")); assert!(order.contains(&"vah264dec")); assert!(order.contains(&"vaapih264dec")); assert!(order.contains(&"v4l2h264dec")); @@ -60,6 +61,19 @@ fn decoder_auto_order_supports_proprietary_and_open_source_routes() { }); } +#[test] +fn vulkan_decoder_fragment_downloads_gpu_memory_before_cpu_sinks() { + let fragment = video_support::h264_decoder_launch_fragment("vulkanh264dec"); + assert!(fragment.contains("vulkanh264dec name=decoder")); + assert!(fragment.contains("discard-corrupted-frames=true")); + assert!(fragment.contains("automatic-request-sync-points=true")); + assert!(fragment.contains("vulkandownload")); + + let named = video_support::h264_decoder_launch_fragment_named("vulkanh264dec", "decoder1"); + assert!(named.contains("vulkanh264dec name=decoder1")); + assert!(named.contains("vulkandownload")); +} + #[test] #[serial] fn decoder_auto_order_can_prefer_software_for_driver_comparisons() { diff --git a/tests/compatibility/downstream/video/downstream_video_mode_decoder_matrix_contract.rs b/tests/compatibility/downstream/video/downstream_video_mode_decoder_matrix_contract.rs index f4587db..c3b2a18 100644 --- a/tests/compatibility/downstream/video/downstream_video_mode_decoder_matrix_contract.rs +++ b/tests/compatibility/downstream/video/downstream_video_mode_decoder_matrix_contract.rs @@ -42,6 +42,7 @@ fn client_decoder_matrix_prefers_known_h264_decoders_with_decodebin_fallback() { "openh264dec", "nvh264dec", "nvh264sldec", + "vulkanh264dec", "vah264dec", "vaapih264dec", "v4l2h264dec", diff --git a/tests/contract/client/input/camera/client_camera_include_contract.rs b/tests/contract/client/input/camera/client_camera_include_contract.rs index 6f09c7d..ad62bfe 100644 --- a/tests/contract/client/input/camera/client_camera_include_contract.rs +++ b/tests/contract/client/input/camera/client_camera_include_contract.rs @@ -98,7 +98,7 @@ mod camera_include_contract { assert!( matches!( enc, - "nvh264enc" | "vaapih264enc" | "v4l2h264enc" | "x264enc" + "nvh264enc" | "vulkanh264enc" | "vaapih264enc" | "v4l2h264enc" | "x264enc" ), "unexpected encoder: {enc}" ); @@ -106,7 +106,7 @@ mod camera_include_contract { assert!( matches!( enc, - "nvh264enc" | "vaapih264enc" | "v4l2h264enc" | "x264enc" + "nvh264enc" | "vulkanh264enc" | "vaapih264enc" | "v4l2h264enc" | "x264enc" ), "unexpected encoder: {enc}" ); @@ -125,8 +125,10 @@ mod camera_include_contract { for expected in [ "\"nvh264enc\" | \"nvh265enc\" if have_nvvidconv", "\"nvh264enc\" | \"nvh265enc\" /* else */", + "\"vulkanh264enc\"", "\"vaapih264enc\" | \"vah265enc\" | \"vaapih265enc\" | \"v4l2h265enc\"", "video/x-raw(memory:NVMM),format=NV12", + "video/x-raw(memory:VulkanImage),format=NV12", "video/x-raw,format=NV12", "video/x-raw,format=I420", ] { @@ -467,6 +469,10 @@ mod camera_include_contract { CameraCapture::encoder_options("nvh264enc", None, 30), "nvh264enc" ); + assert_eq!( + CameraCapture::encoder_options("vulkanh264enc", Some("idr-period"), 30), + "vulkanh264enc bitrate=4500 rate-control=cbr idr-period=30" + ); } #[test] diff --git a/tests/contract/client/output/video/client_output_video_include_contract.rs b/tests/contract/client/output/video/client_output_video_include_contract.rs index 72740c1..c876e6e 100644 --- a/tests/contract/client/output/video/client_output_video_include_contract.rs +++ b/tests/contract/client/output/video/client_output_video_include_contract.rs @@ -135,6 +135,7 @@ mod video_include_contract { let order = h264_decoder_preference_order(); assert_eq!(order.first(), Some(&"nvh264dec")); assert!(order.contains(&"nvh264sldec")); + assert!(order.contains(&"vulkanh264dec")); assert!(order.contains(&"vah264dec")); assert!(order.contains(&"vaapih264dec")); assert!(order.contains(&"v4l2h264dec")); @@ -161,6 +162,19 @@ mod video_include_contract { }); } + #[test] + fn vulkan_decoder_fragment_names_each_decoder_and_downloads_frames() { + let fragment = h264_decoder_launch_fragment("vulkanh264dec"); + assert!(fragment.contains("vulkanh264dec name=decoder")); + assert!(fragment.contains("discard-corrupted-frames=true")); + assert!(fragment.contains("automatic-request-sync-points=true")); + assert!(fragment.contains("vulkandownload")); + + let named = h264_decoder_launch_fragment_named("vulkanh264dec", "decoder1"); + assert!(named.contains("vulkanh264dec name=decoder1")); + assert!(named.contains("vulkandownload")); + } + #[test] #[serial] fn monitor_window_new_covers_x11_backend_path() { diff --git a/tests/installer/scripts/install/client_install_script_contract.rs b/tests/installer/scripts/install/client_install_script_contract.rs index 252fc94..f54c5e7 100644 --- a/tests/installer/scripts/install/client_install_script_contract.rs +++ b/tests/installer/scripts/install/client_install_script_contract.rs @@ -86,11 +86,17 @@ fn client_install_reports_nvidia_and_open_source_media_routes() { "gst_element_available", "first_available_gst_element", "nvidia-smi is available", + "NVIDIA nvcodec is installed but CUDA initialization failed", + "relay profile is H.264", "proprietary NVIDIA GStreamer route", - "open-source VAAPI/V4L2 GStreamer route", + "Vulkan/VAAPI/V4L2 GStreamer route", "nvh265enc", + "nvh264enc", "nvh264dec", "nvh264sldec", + "vulkanh264enc", + "vulkanh264dec", + "vulkanh265dec", "vah265enc", "vaapih265enc", "v4l2h265enc", @@ -99,6 +105,8 @@ fn client_install_reports_nvidia_and_open_source_media_routes() { "v4l2h264dec", "v4l2slh264dec", "x265enc", + "x264enc", + "upstream H.264 encoder candidate", "avdec_h264", "openh264dec", "opusenc",