From e1092afee2a50bab8a809c65a84abc7a9ba70022 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Wed, 15 Apr 2026 12:20:02 -0300 Subject: [PATCH] feat(launcher): use high-quality breakout previews --- client/src/launcher/preview.rs | 270 +++++++++++++++++++++++---- client/src/launcher/ui_components.rs | 18 +- client/src/launcher/ui_runtime.rs | 82 +++++--- 3 files changed, 303 insertions(+), 67 deletions(-) diff --git a/client/src/launcher/preview.rs b/client/src/launcher/preview.rs index 6b7a6ab..516a91a 100644 --- a/client/src/launcher/preview.rs +++ b/client/src/launcher/preview.rs @@ -9,15 +9,15 @@ use gstreamer_app as gst_app; #[cfg(not(coverage))] use gtk::{gdk, glib}; #[cfg(not(coverage))] -use lesavka_common::lesavka::{relay_client::RelayClient, MonitorRequest, VideoPacket}; +use lesavka_common::lesavka::{MonitorRequest, VideoPacket, relay_client::RelayClient}; #[cfg(not(coverage))] -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; #[cfg(not(coverage))] use std::sync::{Arc, Mutex}; #[cfg(not(coverage))] use std::time::Duration; #[cfg(not(coverage))] -use tonic::{transport::Channel, Request}; +use tonic::{Request, transport::Channel}; #[cfg(not(coverage))] use tracing::{debug, warn}; @@ -31,7 +31,8 @@ const PREVIEW_IDLE_STATUS: &str = "Connect relay to preview."; #[cfg(not(coverage))] pub struct LauncherPreview { server_addr: Arc>, - feeds: [PreviewFeed; 2], + inline_feeds: [PreviewFeed; 2], + window_feeds: [PreviewFeed; 2], } #[cfg(not(coverage))] @@ -39,6 +40,40 @@ pub struct LauncherPreview { pub struct PreviewBinding { enabled: Arc, alive: Arc, + active_bindings: Arc, +} + +#[cfg(not(coverage))] +#[derive(Clone, Copy, Debug)] +pub enum PreviewSurface { + Inline, + Window, +} + +#[cfg(not(coverage))] +#[derive(Clone, Copy, Debug)] +struct PreviewProfile { + width: i32, + height: i32, + max_bitrate_kbit: u32, +} + +#[cfg(not(coverage))] +impl PreviewSurface { + fn profile(self) -> PreviewProfile { + match self { + Self::Inline => PreviewProfile { + width: preview_dimension("LESAVKA_PREVIEW_WIDTH", PREVIEW_WIDTH), + height: preview_dimension("LESAVKA_PREVIEW_HEIGHT", PREVIEW_HEIGHT), + max_bitrate_kbit: preview_bitrate("LESAVKA_PREVIEW_MAX_KBIT", 2_500), + }, + Self::Window => PreviewProfile { + width: preview_dimension("LESAVKA_BREAKOUT_PREVIEW_WIDTH", 1280), + height: preview_dimension("LESAVKA_BREAKOUT_PREVIEW_HEIGHT", 720), + max_bitrate_kbit: preview_bitrate("LESAVKA_BREAKOUT_PREVIEW_MAX_KBIT", 8_000), + }, + } + } } #[cfg(not(coverage))] @@ -48,9 +83,13 @@ impl LauncherPreview { let server_addr = Arc::new(Mutex::new(server_addr)); Ok(Self { server_addr: Arc::clone(&server_addr), - feeds: [ - PreviewFeed::spawn(Arc::clone(&server_addr), 0)?, - PreviewFeed::spawn(server_addr, 1)?, + inline_feeds: [ + PreviewFeed::spawn(Arc::clone(&server_addr), 0, PreviewSurface::Inline.profile())?, + PreviewFeed::spawn(server_addr.clone(), 1, PreviewSurface::Inline.profile())?, + ], + window_feeds: [ + PreviewFeed::spawn(Arc::clone(&server_addr), 0, PreviewSurface::Window.profile())?, + PreviewFeed::spawn(server_addr, 1, PreviewSurface::Window.profile())?, ], }) } @@ -62,7 +101,11 @@ impl LauncherPreview { } pub fn set_session_active(&self, active: bool) { - for feed in &self.feeds { + for feed in self + .inline_feeds + .iter() + .chain(self.window_feeds.iter()) + { feed.set_active(active); } } @@ -70,30 +113,53 @@ impl LauncherPreview { pub fn install_on_picture( &self, monitor_id: usize, + surface: PreviewSurface, picture: >k::Picture, status_label: >k::Label, ) -> Option { - self.feeds + self.feeds_for_surface(surface) .get(monitor_id) .map(|feed| feed.install_on_picture(picture, status_label)) } + + fn feeds_for_surface(&self, surface: PreviewSurface) -> &[PreviewFeed; 2] { + match surface { + PreviewSurface::Inline => &self.inline_feeds, + PreviewSurface::Window => &self.window_feeds, + } + } } #[cfg(not(coverage))] impl PreviewBinding { pub fn set_enabled(&self, enabled: bool) { - self.enabled.store(enabled, Ordering::Relaxed); + let was_enabled = self.enabled.swap(enabled, Ordering::AcqRel); + match (was_enabled, enabled) { + (false, true) => { + self.active_bindings.fetch_add(1, Ordering::AcqRel); + } + (true, false) => { + self.active_bindings.fetch_sub(1, Ordering::AcqRel); + } + _ => {} + } } pub fn close(&self) { - self.alive.store(false, Ordering::Relaxed); + if !self.alive.swap(false, Ordering::AcqRel) { + return; + } + if self.enabled.swap(false, Ordering::AcqRel) { + self.active_bindings.fetch_sub(1, Ordering::AcqRel); + } } } #[cfg(not(coverage))] struct PreviewFeed { shared: Arc>, - active: Arc, + session_active: Arc, + active_bindings: Arc, } #[cfg(not(coverage))] @@ -140,21 +206,38 @@ impl SharedPreviewState { #[cfg(not(coverage))] impl PreviewFeed { - fn spawn(server_addr: Arc>, monitor_id: u32) -> Result { + fn spawn( + server_addr: Arc>, + monitor_id: u32, + profile: PreviewProfile, + ) -> Result { let shared = Arc::new(Mutex::new(SharedPreviewState::new())); - let active = Arc::new(AtomicBool::new(false)); + let session_active = Arc::new(AtomicBool::new(false)); + let active_bindings = Arc::new(AtomicUsize::new(0)); let shared_state = Arc::clone(&shared); - let active_flag = Arc::clone(&active); + let session_active_flag = Arc::clone(&session_active); + let active_bindings_flag = Arc::clone(&active_bindings); std::thread::spawn(move || { - if let Err(err) = run_preview_feed(server_addr, monitor_id, active_flag, shared_state) { + if let Err(err) = run_preview_feed( + server_addr, + monitor_id, + profile, + session_active_flag, + active_bindings_flag, + shared_state, + ) { warn!(monitor_id, ?err, "launcher preview feed exited"); } }); - Ok(Self { shared, active }) + Ok(Self { + shared, + session_active, + active_bindings, + }) } fn set_active(&self, active: bool) { - self.active.store(active, Ordering::Relaxed); + self.session_active.store(active, Ordering::Relaxed); if !active { self.replace_status(PREVIEW_IDLE_STATUS, true); } @@ -176,8 +259,10 @@ impl PreviewFeed { let shared = Arc::clone(&self.shared); let enabled = Arc::new(AtomicBool::new(true)); let alive = Arc::new(AtomicBool::new(true)); + let active_bindings = Arc::clone(&self.active_bindings); let enabled_flag = Arc::clone(&enabled); let alive_flag = Arc::clone(&alive); + active_bindings.fetch_add(1, Ordering::AcqRel); let mut last_generation = 0_u64; glib::timeout_add_local(Duration::from_millis(120), move || { if !alive_flag.load(Ordering::Relaxed) { @@ -219,7 +304,11 @@ impl PreviewFeed { } glib::ControlFlow::Continue }); - PreviewBinding { enabled, alive } + PreviewBinding { + enabled, + alive, + active_bindings, + } } } @@ -235,10 +324,12 @@ struct PreviewFrame { fn run_preview_feed( server_addr: Arc>, monitor_id: u32, - active: Arc, + profile: PreviewProfile, + session_active: Arc, + active_bindings: Arc, shared: Arc>, ) -> Result<()> { - let (pipeline, appsrc, appsink) = build_preview_pipeline()?; + let (pipeline, appsrc, appsink) = build_preview_pipeline(profile)?; pipeline .set_state(gst::State::Playing) .context("starting launcher preview pipeline")?; @@ -246,11 +337,13 @@ fn run_preview_feed( { let shared = Arc::clone(&shared); let appsink = appsink.clone(); - std::thread::spawn(move || loop { - if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250)) { - if let Some(frame) = sample_to_frame(&sample) { - if let Ok(mut slot) = shared.lock() { - slot.push_frame(frame); + std::thread::spawn(move || { + loop { + if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250)) { + if let Some(frame) = sample_to_frame(&sample) { + if let Ok(mut slot) = shared.lock() { + slot.push_frame(frame); + } } } } @@ -263,13 +356,25 @@ fn run_preview_feed( .context("building preview tokio runtime")?; let _ = rt.block_on(async move { + let mut was_active = false; + let mut retry_delay = Duration::from_millis(750); loop { - if !active.load(Ordering::Relaxed) { + let active_now = session_active.load(Ordering::Relaxed) + && active_bindings.load(Ordering::Relaxed) > 0; + if !active_now { + was_active = false; + retry_delay = Duration::from_millis(750); set_shared_status(&shared, PREVIEW_IDLE_STATUS, true); tokio::time::sleep(Duration::from_millis(150)).await; continue; } + if !was_active { + was_active = true; + set_shared_status(&shared, "Waking relay preview...", true); + tokio::time::sleep(Duration::from_millis(350)).await; + } + set_shared_status(&shared, "Connecting relay preview...", true); let current_addr = match server_addr.lock() { Ok(value) => value.clone(), @@ -290,14 +395,14 @@ fn run_preview_feed( format!("Preview host is unavailable: {err}"), true, ); - tokio::time::sleep(Duration::from_millis(750)).await; + tokio::time::sleep(retry_delay).await; continue; } }, Err(err) => { warn!(monitor_id, ?err, "launcher preview endpoint invalid"); set_shared_status(&shared, format!("Preview address is invalid: {err}"), true); - tokio::time::sleep(Duration::from_millis(750)).await; + tokio::time::sleep(retry_delay).await; continue; } }; @@ -305,14 +410,17 @@ fn run_preview_feed( let mut cli = RelayClient::new(channel); let req = MonitorRequest { id: monitor_id, - max_bitrate: preview_max_bitrate(), + max_bitrate: profile.max_bitrate_kbit, }; match cli.capture_video(Request::new(req)).await { Ok(mut stream) => { + retry_delay = Duration::from_millis(750); debug!(monitor_id, "launcher preview connected"); set_shared_status(&shared, "Waiting for stream...", true); loop { - if !active.load(Ordering::Relaxed) { + if !session_active.load(Ordering::Relaxed) + || active_bindings.load(Ordering::Relaxed) == 0 + { break; } match tokio::time::timeout( @@ -324,6 +432,7 @@ fn run_preview_feed( Ok(Ok(Some(pkt))) => push_preview_packet(&appsrc, pkt), Ok(Ok(None)) => { set_shared_status(&shared, "Preview stream ended.", true); + retry_delay = Duration::from_millis(1_500); break; } Ok(Err(err)) => { @@ -333,6 +442,7 @@ fn run_preview_feed( format!("Preview stream error: {err}"), true, ); + retry_delay = preview_retry_delay(retry_delay, Some(&err.to_string())); break; } Err(_) => continue, @@ -340,11 +450,22 @@ fn run_preview_feed( } } Err(err) => { - warn!(monitor_id, ?err, "launcher preview rpc failed"); - set_shared_status(&shared, format!("Preview RPC failed: {err}"), true); + if preview_startup_condition(&err) { + debug!( + monitor_id, + ?err, + "launcher preview waiting for capture pipeline" + ); + set_shared_status(&shared, "Waiting for capture pipeline...", true); + retry_delay = preview_retry_delay(retry_delay, Some(err.message())); + } else { + warn!(monitor_id, ?err, "launcher preview rpc failed"); + set_shared_status(&shared, format!("Preview RPC failed: {err}"), true); + retry_delay = preview_retry_delay(retry_delay, Some(err.message())); + } } } - tokio::time::sleep(Duration::from_millis(750)).await; + tokio::time::sleep(retry_delay).await; } #[allow(unreachable_code)] Ok::<(), anyhow::Error>(()) @@ -353,6 +474,38 @@ fn run_preview_feed( Ok(()) } +#[cfg(not(coverage))] +fn preview_startup_condition(err: &tonic::Status) -> bool { + let message = err.message().to_ascii_lowercase(); + err.code() == tonic::Code::Internal + && (message.contains("starting video pipeline") + || message.contains("failed to change its state") + || message.contains("resource busy") + || message.contains("device or resource busy") + || message.contains("no signal")) +} + +#[cfg(not(coverage))] +fn preview_retry_delay(current: Duration, message: Option<&str>) -> Duration { + let current_ms = current.as_millis() as u64; + let mut next_ms = if current_ms < 1_500 { + 1_500 + } else { + current_ms.saturating_mul(2) + }; + if let Some(message) = message { + let message = message.to_ascii_lowercase(); + if message.contains("too many open files") + || message.contains("failed to change its state") + || message.contains("resource busy") + || message.contains("device or resource busy") + { + next_ms = next_ms.max(6_000); + } + } + Duration::from_millis(next_ms.min(30_000)) +} + #[cfg(not(coverage))] fn set_shared_status( shared: &Arc>, @@ -365,13 +518,18 @@ fn set_shared_status( } #[cfg(not(coverage))] -fn build_preview_pipeline() -> Result<(gst::Pipeline, gst_app::AppSrc, gst_app::AppSink)> { +fn build_preview_pipeline( + profile: PreviewProfile, +) -> Result<(gst::Pipeline, gst_app::AppSrc, gst_app::AppSink)> { let desc = format!( "appsrc name=src is-live=true format=time do-timestamp=true block=false ! \ queue max-size-buffers=6 max-size-time=0 max-size-bytes=0 leaky=downstream ! \ h264parse disable-passthrough=true ! avdec_h264 ! videoconvert ! videoscale ! \ - video/x-raw,format=RGBA,width={PREVIEW_WIDTH},height={PREVIEW_HEIGHT},pixel-aspect-ratio=1/1 ! \ + video/x-raw,format=RGBA,width={},height={},pixel-aspect-ratio=1/1 ! \ appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true" + , + profile.width, + profile.height ); let pipeline = gst::parse::launch(&desc)? .downcast::() @@ -398,8 +556,8 @@ fn build_preview_pipeline() -> Result<(gst::Pipeline, gst_app::AppSrc, gst_app:: appsink.set_caps(Some( &gst::Caps::builder("video/x-raw") .field("format", &"RGBA") - .field("width", &PREVIEW_WIDTH) - .field("height", &PREVIEW_HEIGHT) + .field("width", &profile.width) + .field("height", &profile.height) .build(), )); @@ -434,9 +592,39 @@ fn sample_to_frame(sample: &gst::Sample) -> Option { } #[cfg(not(coverage))] -fn preview_max_bitrate() -> u32 { - std::env::var("LESAVKA_PREVIEW_MAX_KBIT") +fn preview_bitrate(var: &str, default: u32) -> u32 { + std::env::var(var) .ok() .and_then(|raw| raw.parse::().ok()) - .unwrap_or(2_500) + .unwrap_or(default) +} + +#[cfg(not(coverage))] +fn preview_dimension(var: &str, default: i32) -> i32 { + std::env::var(var) + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(default) +} + +#[cfg(test)] +mod tests { + use super::{PREVIEW_HEIGHT, PREVIEW_WIDTH, PreviewSurface}; + + #[test] + fn inline_preview_profile_uses_existing_defaults() { + let profile = PreviewSurface::Inline.profile(); + assert_eq!(profile.width, PREVIEW_WIDTH); + assert_eq!(profile.height, PREVIEW_HEIGHT); + assert_eq!(profile.max_bitrate_kbit, 2_500); + } + + #[test] + fn breakout_preview_profile_defaults_to_higher_quality() { + let profile = PreviewSurface::Window.profile(); + assert_eq!(profile.width, 1280); + assert_eq!(profile.height, 720); + assert_eq!(profile.max_bitrate_kbit, 8_000); + } } diff --git a/client/src/launcher/ui_components.rs b/client/src/launcher/ui_components.rs index 724c626..8a502a3 100644 --- a/client/src/launcher/ui_components.rs +++ b/client/src/launcher/ui_components.rs @@ -4,7 +4,7 @@ use gtk::prelude::*; use super::{ devices::DeviceCatalog, - preview::{LauncherPreview, PreviewBinding}, + preview::{LauncherPreview, PreviewBinding, PreviewSurface}, state::LauncherState, }; @@ -403,10 +403,18 @@ pub fn build_launcher_view( let mut left_pane = left_pane; let mut right_pane = right_pane; if let Some(preview) = preview.as_ref() { - left_pane.preview_binding = - preview.install_on_picture(0, &left_pane.picture, &left_pane.stream_status); - right_pane.preview_binding = - preview.install_on_picture(1, &right_pane.picture, &right_pane.stream_status); + left_pane.preview_binding = preview.install_on_picture( + 0, + PreviewSurface::Inline, + &left_pane.picture, + &left_pane.stream_status, + ); + right_pane.preview_binding = preview.install_on_picture( + 1, + PreviewSurface::Inline, + &right_pane.picture, + &right_pane.stream_status, + ); } else { left_pane.stream_status.set_text("Preview unavailable"); right_pane.stream_status.set_text("Preview unavailable"); diff --git a/client/src/launcher/ui_runtime.rs b/client/src/launcher/ui_runtime.rs index a51e8a9..e0cf40c 100644 --- a/client/src/launcher/ui_runtime.rs +++ b/client/src/launcher/ui_runtime.rs @@ -8,10 +8,12 @@ use std::{ }; use super::{ + LAUNCHER_CLIPBOARD_CONTROL_ENV, LAUNCHER_FOCUS_SIGNAL_ENV, device_test::{DeviceTestController, DeviceTestKind}, + launcher_clipboard_control_path, launcher_focus_signal_path, - preview::LauncherPreview, + preview::{LauncherPreview, PreviewSurface}, runtime_env_vars, state::{CapturePowerStatus, DisplaySurface, InputRouting, LauncherState}, ui_components::{DisplayPaneWidgets, LauncherWidgets, PopoutWindowHandle}, @@ -159,7 +161,11 @@ pub fn open_popout_window( widgets: &LauncherWidgets, monitor_id: usize, ) { - if popouts.borrow()[monitor_id].is_some() { + let already_open = { + let popouts = popouts.borrow(); + popouts[monitor_id].is_some() + }; + if already_open { return; } @@ -201,7 +207,12 @@ pub fn open_popout_window( root.append(&stream_status); let binding = preview - .install_on_picture(monitor_id, &picture, &stream_status) + .install_on_picture( + monitor_id, + PreviewSurface::Window, + &picture, + &stream_status, + ) .expect("preview binding for popout"); window.set_child(Some(&root)); @@ -224,13 +235,16 @@ pub fn open_popout_window( { preview_binding.set_enabled(true); } - state_handle - .borrow_mut() - .set_display_surface(monitor_id, DisplaySurface::Preview); + { + let mut state = state_handle.borrow_mut(); + state.set_display_surface(monitor_id, DisplaySurface::Preview); + } + let child_running = child_proc_handle.borrow().is_some(); + let state_snapshot = state_handle.borrow().clone(); refresh_launcher_ui( &widgets_handle, - &state_handle.borrow(), - child_proc_handle.borrow().is_some(), + &state_snapshot, + child_running, ); } else { close_binding.close(); @@ -238,14 +252,20 @@ pub fn open_popout_window( glib::Propagation::Proceed }); - state - .borrow_mut() - .set_display_surface(monitor_id, DisplaySurface::Window); - popouts.borrow_mut()[monitor_id] = Some(PopoutWindowHandle { - window: window.clone(), - binding, - }); - refresh_launcher_ui(widgets, &state.borrow(), child_proc.borrow().is_some()); + { + let mut state = state.borrow_mut(); + state.set_display_surface(monitor_id, DisplaySurface::Window); + } + { + let mut popouts = popouts.borrow_mut(); + popouts[monitor_id] = Some(PopoutWindowHandle { + window: window.clone(), + binding, + }); + } + let child_running = child_proc.borrow().is_some(); + let state_snapshot = state.borrow().clone(); + refresh_launcher_ui(widgets, &state_snapshot, child_running); window.present(); } @@ -267,10 +287,13 @@ pub fn dock_display_to_preview( if let Some(binding) = widgets.display_panes[monitor_id].preview_binding.as_ref() { binding.set_enabled(true); } - state - .borrow_mut() - .set_display_surface(monitor_id, DisplaySurface::Preview); - refresh_launcher_ui(widgets, &state.borrow(), child_proc.borrow().is_some()); + { + let mut state = state.borrow_mut(); + state.set_display_surface(monitor_id, DisplaySurface::Preview); + } + let child_running = child_proc.borrow().is_some(); + let state_snapshot = state.borrow().clone(); + refresh_launcher_ui(widgets, &state_snapshot, child_running); } pub fn refresh_display_pane(pane: &DisplayPaneWidgets, surface: DisplaySurface) { @@ -512,6 +535,19 @@ pub fn input_state_path() -> PathBuf { .unwrap_or_else(|_| PathBuf::from(DEFAULT_INPUT_STATE_PATH)) } +pub fn write_clipboard_control_request(path: &Path) -> Result<()> { + std::fs::write( + path, + format!( + "{}\n", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis() + ), + )?; + Ok(()) +} + pub fn write_input_routing_request(path: &Path, routing: InputRouting) -> Result<()> { std::fs::write(path, format!("{}\n", routing_name(routing)))?; Ok(()) @@ -639,10 +675,14 @@ pub fn spawn_client_process( command.env("LESAVKA_LAUNCHER_WINDOW_TITLE", "Lesavka Launcher"); command.env("LESAVKA_FOCUS_LAUNCHER_ON_LOCAL", "1"); command.env(LAUNCHER_FOCUS_SIGNAL_ENV, launcher_focus_signal_path()); + command.env( + LAUNCHER_CLIPBOARD_CONTROL_ENV, + launcher_clipboard_control_path(), + ); command.env(INPUT_CONTROL_ENV, input_control_path); command.env(INPUT_STATE_ENV, input_state_path); command.env("LESAVKA_DISABLE_VIDEO_RENDER", "1"); - command.env("LESAVKA_CLIPBOARD_PASTE", "0"); + command.env("LESAVKA_CLIPBOARD_PASTE", "1"); for (key, value) in runtime_env_vars(state) { command.env(key, value); }