fix(camera): make webcam preview track send quality

This commit is contained in:
Brad Stein 2026-04-23 04:46:21 -03:00
parent 7e780ffaf0
commit ca10c667f5
3 changed files with 102 additions and 28 deletions

View File

@ -16,8 +16,6 @@ use std::{
}; };
const CAMERA_PREVIEW_TAP_ENV: &str = "LESAVKA_UPLINK_CAMERA_PREVIEW"; const CAMERA_PREVIEW_TAP_ENV: &str = "LESAVKA_UPLINK_CAMERA_PREVIEW";
const CAMERA_PREVIEW_WIDTH: i32 = 128;
const CAMERA_PREVIEW_HEIGHT: i32 = 72;
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CameraSourceProfile { enum CameraSourceProfile {
@ -161,6 +159,7 @@ impl CameraCapture {
// * vaapih264enc wants system-memory caps; // * vaapih264enc wants system-memory caps;
// * x264enc needs the usual raw caps. // * x264enc needs the usual raw caps.
let preview_tap_path = camera_preview_tap_path(); let preview_tap_path = camera_preview_tap_path();
let preview_tap_branch = camera_preview_tap_branch(width, height, fps);
let raw_source_chain = let raw_source_chain =
camera_raw_source_chain(&src_desc, &src_caps, width, height, fps, source_profile); camera_raw_source_chain(&src_desc, &src_caps, width, height, fps, source_profile);
let desc = if preview_tap_path.is_some() { let desc = if preview_tap_path.is_some() {
@ -173,8 +172,7 @@ impl CameraCapture {
t. ! queue max-size-buffers=30 leaky=downstream ! \ t. ! queue max-size-buffers=30 leaky=downstream ! \
appsink name=asink emit-signals=true max-buffers=60 drop=true \ appsink name=asink emit-signals=true max-buffers=60 drop=true \
t. ! queue max-size-buffers=2 leaky=downstream ! jpegdec ! \ t. ! queue max-size-buffers=2 leaky=downstream ! jpegdec ! \
{}", {preview_tap_branch}"
camera_preview_tap_branch()
) )
} else { } else {
format!( format!(
@ -184,8 +182,7 @@ impl CameraCapture {
videoconvert ! jpegenc quality={jpeg_quality} ! \ videoconvert ! jpegenc quality={jpeg_quality} ! \
appsink name=asink emit-signals=true max-buffers=60 drop=true \ appsink name=asink emit-signals=true max-buffers=60 drop=true \
t. ! queue max-size-buffers=2 leaky=downstream ! \ t. ! queue max-size-buffers=2 leaky=downstream ! \
{}", {preview_tap_branch}"
camera_preview_tap_branch()
) )
} }
} else if use_mjpg_source { } else if use_mjpg_source {
@ -199,8 +196,7 @@ impl CameraCapture {
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \ h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
appsink name=asink emit-signals=true max-buffers=60 drop=true \ appsink name=asink emit-signals=true max-buffers=60 drop=true \
t. ! queue max-size-buffers=2 leaky=downstream ! \ t. ! queue max-size-buffers=2 leaky=downstream ! \
{}", {preview_tap_branch}"
camera_preview_tap_branch()
) )
} else { } else {
format!( format!(
@ -211,8 +207,7 @@ impl CameraCapture {
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \ h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
appsink name=asink emit-signals=true max-buffers=60 drop=true \ appsink name=asink emit-signals=true max-buffers=60 drop=true \
t. ! queue max-size-buffers=2 leaky=downstream ! \ t. ! queue max-size-buffers=2 leaky=downstream ! \
{}", {preview_tap_branch}"
camera_preview_tap_branch()
) )
} }
} else if output_mjpeg { } else if output_mjpeg {
@ -558,15 +553,18 @@ fn camera_preview_tap_path() -> Option<PathBuf> {
.map(PathBuf::from) .map(PathBuf::from)
} }
fn camera_preview_tap_branch() -> String { fn camera_preview_tap_branch(width: u32, height: u32, fps: u32) -> String {
let preview_width = width.clamp(1, i32::MAX as u32);
let preview_height = height.clamp(1, i32::MAX as u32);
let preview_fps = fps.clamp(1, 60);
format!( format!(
"videoconvert ! videoscale ! videorate ! \ "videoconvert ! videoscale ! videorate ! \
video/x-raw,format=RGBA,width={CAMERA_PREVIEW_WIDTH},height={CAMERA_PREVIEW_HEIGHT},framerate=10/1,pixel-aspect-ratio=1/1 ! \ video/x-raw,format=RGBA,width={preview_width},height={preview_height},framerate={preview_fps}/1,pixel-aspect-ratio=1/1 ! \
appsink name=preview_sink emit-signals=false sync=false max-buffers=1 drop=true" appsink name=preview_sink emit-signals=false sync=false max-buffers=1 drop=true"
) )
} }
/// Publish tiny local preview frames so the launcher can prove uplink activity. /// Publish actual-size local preview frames so the launcher mirrors uplink quality.
fn spawn_camera_preview_tap(sink: gst_app::AppSink, path: PathBuf) -> Arc<AtomicBool> { fn spawn_camera_preview_tap(sink: gst_app::AppSink, path: PathBuf) -> Arc<AtomicBool> {
let running = Arc::new(AtomicBool::new(true)); let running = Arc::new(AtomicBool::new(true));
let thread_running = Arc::clone(&running); let thread_running = Arc::clone(&running);

View File

@ -14,8 +14,9 @@ use std::time::Duration;
use super::devices::CameraMode; use super::devices::CameraMode;
const CAMERA_PREVIEW_WIDTH: i32 = 128; const CAMERA_PREVIEW_DEFAULT_WIDTH: i32 = 1280;
const CAMERA_PREVIEW_HEIGHT: i32 = 72; const CAMERA_PREVIEW_DEFAULT_HEIGHT: i32 = 720;
const CAMERA_PREVIEW_DEFAULT_FPS: u32 = 30;
const CAMERA_PREVIEW_IDLE: &str = "Select a webcam and click Start Preview."; const CAMERA_PREVIEW_IDLE: &str = "Select a webcam and click Start Preview.";
const MIC_MONITOR_RATE: i32 = 16_000; const MIC_MONITOR_RATE: i32 = 16_000;
const MIC_MONITOR_CHANNELS: i32 = 1; const MIC_MONITOR_CHANNELS: i32 = 1;
@ -611,14 +612,15 @@ impl LocalCameraPreview {
} }
fn blank_camera_preview_texture() -> gdk::MemoryTexture { fn blank_camera_preview_texture() -> gdk::MemoryTexture {
let rgba = vec![12_u8; (CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 4) as usize]; let rgba =
vec![12_u8; (CAMERA_PREVIEW_DEFAULT_WIDTH * CAMERA_PREVIEW_DEFAULT_HEIGHT * 4) as usize];
let bytes = glib::Bytes::from_owned(rgba); let bytes = glib::Bytes::from_owned(rgba);
gdk::MemoryTexture::new( gdk::MemoryTexture::new(
CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_DEFAULT_WIDTH,
CAMERA_PREVIEW_HEIGHT, CAMERA_PREVIEW_DEFAULT_HEIGHT,
gdk::MemoryFormat::R8g8b8a8, gdk::MemoryFormat::R8g8b8a8,
&bytes, &bytes,
(CAMERA_PREVIEW_WIDTH * 4) as usize, (CAMERA_PREVIEW_DEFAULT_WIDTH * 4) as usize,
) )
} }
@ -786,15 +788,30 @@ fn run_camera_preview_feed(
let quality = mode let quality = mode
.map(CameraMode::short_label) .map(CameraMode::short_label)
.unwrap_or_else(|| "default quality".to_string()); .unwrap_or_else(|| "default quality".to_string());
*status = format!("Local preview live for {selected} at {quality}."); *status = format!("Local preview live for {selected} at {quality}; waiting for frames...");
} }
let mut announced_size = None::<(i32, i32)>;
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token { while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250)) if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250))
&& let Some(frame) = sample_to_frame(&sample) && let Some(frame) = sample_to_frame(&sample)
&& let Ok(mut slot) = latest.lock()
{ {
*slot = Some(frame); let size = (frame.width, frame.height);
if announced_size != Some(size) {
announced_size = Some(size);
if let Ok(mut status) = status_text.lock() {
let quality = mode
.map(CameraMode::short_label)
.unwrap_or_else(|| "default quality".to_string());
*status = format!(
"Local preview live for {selected} at {quality}; showing {}x{}.",
size.0, size.1
);
}
}
if let Ok(mut slot) = latest.lock() {
*slot = Some(frame);
}
} }
} }
@ -812,16 +829,22 @@ fn run_camera_file_preview_feed(
running: Arc<AtomicBool>, running: Arc<AtomicBool>,
) { ) {
let mut has_frame = false; let mut has_frame = false;
let mut announced_size = None::<(i32, i32)>;
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token { while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
match read_camera_preview_tap(&path) { match read_camera_preview_tap(&path) {
Ok(frame) => { Ok(frame) => {
let size = (frame.width, frame.height);
if let Ok(mut slot) = latest.lock() { if let Ok(mut slot) = latest.lock() {
*slot = Some(frame); *slot = Some(frame);
} }
if !has_frame { if !has_frame || announced_size != Some(size) {
has_frame = true; has_frame = true;
announced_size = Some(size);
if let Ok(mut status) = status_text.lock() { if let Ok(mut status) = status_text.lock() {
*status = format!("Relay webcam preview live for {selected}."); *status = format!(
"Relay webcam preview live for {selected}; showing {}x{}.",
size.0, size.1
);
} }
} }
} }
@ -866,6 +889,7 @@ fn build_camera_preview_pipeline(
mode: Option<CameraMode>, mode: Option<CameraMode>,
) -> Result<(gst::Pipeline, gst_app::AppSink)> { ) -> Result<(gst::Pipeline, gst_app::AppSink)> {
let desc = camera_preview_pipeline_desc(device, mode); let desc = camera_preview_pipeline_desc(device, mode);
let (width, height, _fps) = camera_preview_mode(mode);
let pipeline = gst::parse::launch(&desc)? let pipeline = gst::parse::launch(&desc)?
.downcast::<gst::Pipeline>() .downcast::<gst::Pipeline>()
.expect("camera preview pipeline"); .expect("camera preview pipeline");
@ -877,8 +901,8 @@ fn build_camera_preview_pipeline(
appsink.set_caps(Some( appsink.set_caps(Some(
&gst::Caps::builder("video/x-raw") &gst::Caps::builder("video/x-raw")
.field("format", "RGBA") .field("format", "RGBA")
.field("width", CAMERA_PREVIEW_WIDTH) .field("width", width)
.field("height", CAMERA_PREVIEW_HEIGHT) .field("height", height)
.build(), .build(),
)); ));
Ok((pipeline, appsink)) Ok((pipeline, appsink))
@ -909,6 +933,7 @@ fn build_microphone_monitor_pipeline(
fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> String { fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> String {
let device = gst_quote(device); let device = gst_quote(device);
let (preview_width, preview_height, preview_fps) = camera_preview_mode(mode);
let source_caps = mode let source_caps = mode
.map(|mode| { .map(|mode| {
format!( format!(
@ -920,11 +945,26 @@ fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> Strin
format!( format!(
"v4l2src device=\"{device}\" do-timestamp=true ! \ "v4l2src device=\"{device}\" do-timestamp=true ! \
{source_caps}videoconvert ! videoscale ! videorate ! \ {source_caps}videoconvert ! videoscale ! videorate ! \
video/x-raw,format=RGBA,width={CAMERA_PREVIEW_WIDTH},height={CAMERA_PREVIEW_HEIGHT},framerate=30/1,pixel-aspect-ratio=1/1 ! \ video/x-raw,format=RGBA,width={preview_width},height={preview_height},framerate={preview_fps}/1,pixel-aspect-ratio=1/1 ! \
appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true" appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true"
) )
} }
fn camera_preview_mode(mode: Option<CameraMode>) -> (i32, i32, u32) {
mode.map(|mode| {
(
i32::try_from(mode.width).unwrap_or(i32::MAX).max(1),
i32::try_from(mode.height).unwrap_or(i32::MAX).max(1),
mode.fps.max(1),
)
})
.unwrap_or((
CAMERA_PREVIEW_DEFAULT_WIDTH,
CAMERA_PREVIEW_DEFAULT_HEIGHT,
CAMERA_PREVIEW_DEFAULT_FPS,
))
}
fn microphone_monitor_pipeline_desc(source: &str, sink: Option<&str>) -> String { fn microphone_monitor_pipeline_desc(source: &str, sink: Option<&str>) -> String {
let source_element = if looks_like_pulse_source_name(source) let source_element = if looks_like_pulse_source_name(source)
|| gst::ElementFactory::find("pipewiresrc").is_none() || gst::ElementFactory::find("pipewiresrc").is_none()
@ -1106,7 +1146,7 @@ fn build_wav_bytes(audio: &[u8], sample_rate: u32, channels: u16, bits_per_sampl
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ use super::{
MIC_REPLAY_MAX_BYTES, build_wav_bytes, camera_preview_pipeline_desc, MIC_REPLAY_MAX_BYTES, build_wav_bytes, camera_preview_mode, camera_preview_pipeline_desc,
microphone_monitor_pipeline_desc, normalize_camera_selection, push_recent_audio, microphone_monitor_pipeline_desc, normalize_camera_selection, push_recent_audio,
read_camera_preview_tap, read_microphone_level_tap, resolve_camera_device, read_camera_preview_tap, read_microphone_level_tap, resolve_camera_device,
}; };
@ -1152,6 +1192,17 @@ mod tests {
desc.contains("image/jpeg,width=(int)1920,height=(int)1080,framerate=(fraction)30/1") desc.contains("image/jpeg,width=(int)1920,height=(int)1080,framerate=(fraction)30/1")
); );
assert!(desc.contains("decodebin ! videoconvert ! videoscale")); assert!(desc.contains("decodebin ! videoconvert ! videoscale"));
assert!(desc.contains("video/x-raw,format=RGBA,width=1920,height=1080,framerate=30/1"));
assert!(!desc.contains("width=128,height=72"));
}
#[test]
fn camera_preview_mode_defaults_to_hd_and_tracks_selected_quality() {
assert_eq!(camera_preview_mode(None), (1280, 720, 30));
assert_eq!(
camera_preview_mode(Some(CameraMode::new(1920, 1080, 30))),
(1920, 1080, 30)
);
} }
#[test] #[test]

View File

@ -68,6 +68,21 @@ mod camera_include_contract {
assert!(chain.contains("capsfilter caps=\"")); assert!(chain.contains("capsfilter caps=\""));
} }
#[test]
fn camera_preview_tap_uses_the_actual_uplink_dimensions() {
let branch = camera_preview_tap_branch(1920, 1080, 30);
assert!(branch.contains("width=1920,height=1080"));
assert!(branch.contains("framerate=30/1"));
assert!(!branch.contains("width=128,height=72"));
let branch = camera_preview_tap_branch(1280, 720, 120);
assert!(branch.contains("width=1280,height=720"));
assert!(
branch.contains("framerate=60/1"),
"preview tap should cap frame production while preserving resolution"
);
}
#[test] #[test]
fn encoder_helpers_return_supported_defaults() { fn encoder_helpers_return_supported_defaults() {
init_gst(); init_gst();
@ -202,6 +217,16 @@ mod camera_include_contract {
bytes.starts_with(b"LESAVKA_RGBA "), bytes.starts_with(b"LESAVKA_RGBA "),
"preview tap should publish an RGBA frame header" "preview tap should publish an RGBA frame header"
); );
let header_end = bytes
.iter()
.position(|byte| *byte == b'\n')
.expect("preview header newline");
let header =
std::str::from_utf8(&bytes[..header_end]).expect("utf8 header");
assert!(
header.starts_with("LESAVKA_RGBA 160 90 "),
"preview tap should prove the selected uplink size, got {header:?}"
);
return; return;
} }
std::thread::sleep(std::time::Duration::from_millis(50)); std::thread::sleep(std::time::Duration::from_millis(50));