fix(camera): make webcam preview track send quality
This commit is contained in:
parent
7e780ffaf0
commit
ca10c667f5
@ -16,8 +16,6 @@ use std::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
const CAMERA_PREVIEW_TAP_ENV: &str = "LESAVKA_UPLINK_CAMERA_PREVIEW";
|
const CAMERA_PREVIEW_TAP_ENV: &str = "LESAVKA_UPLINK_CAMERA_PREVIEW";
|
||||||
const CAMERA_PREVIEW_WIDTH: i32 = 128;
|
|
||||||
const CAMERA_PREVIEW_HEIGHT: i32 = 72;
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
enum CameraSourceProfile {
|
enum CameraSourceProfile {
|
||||||
@ -161,6 +159,7 @@ impl CameraCapture {
|
|||||||
// * vaapih264enc wants system-memory caps;
|
// * vaapih264enc wants system-memory caps;
|
||||||
// * x264enc needs the usual raw caps.
|
// * x264enc needs the usual raw caps.
|
||||||
let preview_tap_path = camera_preview_tap_path();
|
let preview_tap_path = camera_preview_tap_path();
|
||||||
|
let preview_tap_branch = camera_preview_tap_branch(width, height, fps);
|
||||||
let raw_source_chain =
|
let raw_source_chain =
|
||||||
camera_raw_source_chain(&src_desc, &src_caps, width, height, fps, source_profile);
|
camera_raw_source_chain(&src_desc, &src_caps, width, height, fps, source_profile);
|
||||||
let desc = if preview_tap_path.is_some() {
|
let desc = if preview_tap_path.is_some() {
|
||||||
@ -173,8 +172,7 @@ impl CameraCapture {
|
|||||||
t. ! queue max-size-buffers=30 leaky=downstream ! \
|
t. ! queue max-size-buffers=30 leaky=downstream ! \
|
||||||
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
||||||
t. ! queue max-size-buffers=2 leaky=downstream ! jpegdec ! \
|
t. ! queue max-size-buffers=2 leaky=downstream ! jpegdec ! \
|
||||||
{}",
|
{preview_tap_branch}"
|
||||||
camera_preview_tap_branch()
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
format!(
|
format!(
|
||||||
@ -184,8 +182,7 @@ impl CameraCapture {
|
|||||||
videoconvert ! jpegenc quality={jpeg_quality} ! \
|
videoconvert ! jpegenc quality={jpeg_quality} ! \
|
||||||
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
||||||
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
||||||
{}",
|
{preview_tap_branch}"
|
||||||
camera_preview_tap_branch()
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else if use_mjpg_source {
|
} else if use_mjpg_source {
|
||||||
@ -199,8 +196,7 @@ impl CameraCapture {
|
|||||||
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
|
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
|
||||||
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
||||||
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
||||||
{}",
|
{preview_tap_branch}"
|
||||||
camera_preview_tap_branch()
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
format!(
|
format!(
|
||||||
@ -211,8 +207,7 @@ impl CameraCapture {
|
|||||||
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
|
h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! \
|
||||||
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
appsink name=asink emit-signals=true max-buffers=60 drop=true \
|
||||||
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
t. ! queue max-size-buffers=2 leaky=downstream ! \
|
||||||
{}",
|
{preview_tap_branch}"
|
||||||
camera_preview_tap_branch()
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else if output_mjpeg {
|
} else if output_mjpeg {
|
||||||
@ -558,15 +553,18 @@ fn camera_preview_tap_path() -> Option<PathBuf> {
|
|||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn camera_preview_tap_branch() -> String {
|
fn camera_preview_tap_branch(width: u32, height: u32, fps: u32) -> String {
|
||||||
|
let preview_width = width.clamp(1, i32::MAX as u32);
|
||||||
|
let preview_height = height.clamp(1, i32::MAX as u32);
|
||||||
|
let preview_fps = fps.clamp(1, 60);
|
||||||
format!(
|
format!(
|
||||||
"videoconvert ! videoscale ! videorate ! \
|
"videoconvert ! videoscale ! videorate ! \
|
||||||
video/x-raw,format=RGBA,width={CAMERA_PREVIEW_WIDTH},height={CAMERA_PREVIEW_HEIGHT},framerate=10/1,pixel-aspect-ratio=1/1 ! \
|
video/x-raw,format=RGBA,width={preview_width},height={preview_height},framerate={preview_fps}/1,pixel-aspect-ratio=1/1 ! \
|
||||||
appsink name=preview_sink emit-signals=false sync=false max-buffers=1 drop=true"
|
appsink name=preview_sink emit-signals=false sync=false max-buffers=1 drop=true"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Publish tiny local preview frames so the launcher can prove uplink activity.
|
/// Publish actual-size local preview frames so the launcher mirrors uplink quality.
|
||||||
fn spawn_camera_preview_tap(sink: gst_app::AppSink, path: PathBuf) -> Arc<AtomicBool> {
|
fn spawn_camera_preview_tap(sink: gst_app::AppSink, path: PathBuf) -> Arc<AtomicBool> {
|
||||||
let running = Arc::new(AtomicBool::new(true));
|
let running = Arc::new(AtomicBool::new(true));
|
||||||
let thread_running = Arc::clone(&running);
|
let thread_running = Arc::clone(&running);
|
||||||
|
|||||||
@ -14,8 +14,9 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use super::devices::CameraMode;
|
use super::devices::CameraMode;
|
||||||
|
|
||||||
const CAMERA_PREVIEW_WIDTH: i32 = 128;
|
const CAMERA_PREVIEW_DEFAULT_WIDTH: i32 = 1280;
|
||||||
const CAMERA_PREVIEW_HEIGHT: i32 = 72;
|
const CAMERA_PREVIEW_DEFAULT_HEIGHT: i32 = 720;
|
||||||
|
const CAMERA_PREVIEW_DEFAULT_FPS: u32 = 30;
|
||||||
const CAMERA_PREVIEW_IDLE: &str = "Select a webcam and click Start Preview.";
|
const CAMERA_PREVIEW_IDLE: &str = "Select a webcam and click Start Preview.";
|
||||||
const MIC_MONITOR_RATE: i32 = 16_000;
|
const MIC_MONITOR_RATE: i32 = 16_000;
|
||||||
const MIC_MONITOR_CHANNELS: i32 = 1;
|
const MIC_MONITOR_CHANNELS: i32 = 1;
|
||||||
@ -611,14 +612,15 @@ impl LocalCameraPreview {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn blank_camera_preview_texture() -> gdk::MemoryTexture {
|
fn blank_camera_preview_texture() -> gdk::MemoryTexture {
|
||||||
let rgba = vec![12_u8; (CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 4) as usize];
|
let rgba =
|
||||||
|
vec![12_u8; (CAMERA_PREVIEW_DEFAULT_WIDTH * CAMERA_PREVIEW_DEFAULT_HEIGHT * 4) as usize];
|
||||||
let bytes = glib::Bytes::from_owned(rgba);
|
let bytes = glib::Bytes::from_owned(rgba);
|
||||||
gdk::MemoryTexture::new(
|
gdk::MemoryTexture::new(
|
||||||
CAMERA_PREVIEW_WIDTH,
|
CAMERA_PREVIEW_DEFAULT_WIDTH,
|
||||||
CAMERA_PREVIEW_HEIGHT,
|
CAMERA_PREVIEW_DEFAULT_HEIGHT,
|
||||||
gdk::MemoryFormat::R8g8b8a8,
|
gdk::MemoryFormat::R8g8b8a8,
|
||||||
&bytes,
|
&bytes,
|
||||||
(CAMERA_PREVIEW_WIDTH * 4) as usize,
|
(CAMERA_PREVIEW_DEFAULT_WIDTH * 4) as usize,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -786,15 +788,30 @@ fn run_camera_preview_feed(
|
|||||||
let quality = mode
|
let quality = mode
|
||||||
.map(CameraMode::short_label)
|
.map(CameraMode::short_label)
|
||||||
.unwrap_or_else(|| "default quality".to_string());
|
.unwrap_or_else(|| "default quality".to_string());
|
||||||
*status = format!("Local preview live for {selected} at {quality}.");
|
*status = format!("Local preview live for {selected} at {quality}; waiting for frames...");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut announced_size = None::<(i32, i32)>;
|
||||||
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
|
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
|
||||||
if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250))
|
if let Some(sample) = appsink.try_pull_sample(gst::ClockTime::from_mseconds(250))
|
||||||
&& let Some(frame) = sample_to_frame(&sample)
|
&& let Some(frame) = sample_to_frame(&sample)
|
||||||
&& let Ok(mut slot) = latest.lock()
|
|
||||||
{
|
{
|
||||||
*slot = Some(frame);
|
let size = (frame.width, frame.height);
|
||||||
|
if announced_size != Some(size) {
|
||||||
|
announced_size = Some(size);
|
||||||
|
if let Ok(mut status) = status_text.lock() {
|
||||||
|
let quality = mode
|
||||||
|
.map(CameraMode::short_label)
|
||||||
|
.unwrap_or_else(|| "default quality".to_string());
|
||||||
|
*status = format!(
|
||||||
|
"Local preview live for {selected} at {quality}; showing {}x{}.",
|
||||||
|
size.0, size.1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(mut slot) = latest.lock() {
|
||||||
|
*slot = Some(frame);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -812,16 +829,22 @@ fn run_camera_file_preview_feed(
|
|||||||
running: Arc<AtomicBool>,
|
running: Arc<AtomicBool>,
|
||||||
) {
|
) {
|
||||||
let mut has_frame = false;
|
let mut has_frame = false;
|
||||||
|
let mut announced_size = None::<(i32, i32)>;
|
||||||
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
|
while running.load(Ordering::Acquire) && generation.load(Ordering::Acquire) == token {
|
||||||
match read_camera_preview_tap(&path) {
|
match read_camera_preview_tap(&path) {
|
||||||
Ok(frame) => {
|
Ok(frame) => {
|
||||||
|
let size = (frame.width, frame.height);
|
||||||
if let Ok(mut slot) = latest.lock() {
|
if let Ok(mut slot) = latest.lock() {
|
||||||
*slot = Some(frame);
|
*slot = Some(frame);
|
||||||
}
|
}
|
||||||
if !has_frame {
|
if !has_frame || announced_size != Some(size) {
|
||||||
has_frame = true;
|
has_frame = true;
|
||||||
|
announced_size = Some(size);
|
||||||
if let Ok(mut status) = status_text.lock() {
|
if let Ok(mut status) = status_text.lock() {
|
||||||
*status = format!("Relay webcam preview live for {selected}.");
|
*status = format!(
|
||||||
|
"Relay webcam preview live for {selected}; showing {}x{}.",
|
||||||
|
size.0, size.1
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -866,6 +889,7 @@ fn build_camera_preview_pipeline(
|
|||||||
mode: Option<CameraMode>,
|
mode: Option<CameraMode>,
|
||||||
) -> Result<(gst::Pipeline, gst_app::AppSink)> {
|
) -> Result<(gst::Pipeline, gst_app::AppSink)> {
|
||||||
let desc = camera_preview_pipeline_desc(device, mode);
|
let desc = camera_preview_pipeline_desc(device, mode);
|
||||||
|
let (width, height, _fps) = camera_preview_mode(mode);
|
||||||
let pipeline = gst::parse::launch(&desc)?
|
let pipeline = gst::parse::launch(&desc)?
|
||||||
.downcast::<gst::Pipeline>()
|
.downcast::<gst::Pipeline>()
|
||||||
.expect("camera preview pipeline");
|
.expect("camera preview pipeline");
|
||||||
@ -877,8 +901,8 @@ fn build_camera_preview_pipeline(
|
|||||||
appsink.set_caps(Some(
|
appsink.set_caps(Some(
|
||||||
&gst::Caps::builder("video/x-raw")
|
&gst::Caps::builder("video/x-raw")
|
||||||
.field("format", "RGBA")
|
.field("format", "RGBA")
|
||||||
.field("width", CAMERA_PREVIEW_WIDTH)
|
.field("width", width)
|
||||||
.field("height", CAMERA_PREVIEW_HEIGHT)
|
.field("height", height)
|
||||||
.build(),
|
.build(),
|
||||||
));
|
));
|
||||||
Ok((pipeline, appsink))
|
Ok((pipeline, appsink))
|
||||||
@ -909,6 +933,7 @@ fn build_microphone_monitor_pipeline(
|
|||||||
|
|
||||||
fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> String {
|
fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> String {
|
||||||
let device = gst_quote(device);
|
let device = gst_quote(device);
|
||||||
|
let (preview_width, preview_height, preview_fps) = camera_preview_mode(mode);
|
||||||
let source_caps = mode
|
let source_caps = mode
|
||||||
.map(|mode| {
|
.map(|mode| {
|
||||||
format!(
|
format!(
|
||||||
@ -920,11 +945,26 @@ fn camera_preview_pipeline_desc(device: &str, mode: Option<CameraMode>) -> Strin
|
|||||||
format!(
|
format!(
|
||||||
"v4l2src device=\"{device}\" do-timestamp=true ! \
|
"v4l2src device=\"{device}\" do-timestamp=true ! \
|
||||||
{source_caps}videoconvert ! videoscale ! videorate ! \
|
{source_caps}videoconvert ! videoscale ! videorate ! \
|
||||||
video/x-raw,format=RGBA,width={CAMERA_PREVIEW_WIDTH},height={CAMERA_PREVIEW_HEIGHT},framerate=30/1,pixel-aspect-ratio=1/1 ! \
|
video/x-raw,format=RGBA,width={preview_width},height={preview_height},framerate={preview_fps}/1,pixel-aspect-ratio=1/1 ! \
|
||||||
appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true"
|
appsink name=sink emit-signals=false sync=false max-buffers=1 drop=true"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn camera_preview_mode(mode: Option<CameraMode>) -> (i32, i32, u32) {
|
||||||
|
mode.map(|mode| {
|
||||||
|
(
|
||||||
|
i32::try_from(mode.width).unwrap_or(i32::MAX).max(1),
|
||||||
|
i32::try_from(mode.height).unwrap_or(i32::MAX).max(1),
|
||||||
|
mode.fps.max(1),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or((
|
||||||
|
CAMERA_PREVIEW_DEFAULT_WIDTH,
|
||||||
|
CAMERA_PREVIEW_DEFAULT_HEIGHT,
|
||||||
|
CAMERA_PREVIEW_DEFAULT_FPS,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn microphone_monitor_pipeline_desc(source: &str, sink: Option<&str>) -> String {
|
fn microphone_monitor_pipeline_desc(source: &str, sink: Option<&str>) -> String {
|
||||||
let source_element = if looks_like_pulse_source_name(source)
|
let source_element = if looks_like_pulse_source_name(source)
|
||||||
|| gst::ElementFactory::find("pipewiresrc").is_none()
|
|| gst::ElementFactory::find("pipewiresrc").is_none()
|
||||||
@ -1106,7 +1146,7 @@ fn build_wav_bytes(audio: &[u8], sample_rate: u32, channels: u16, bits_per_sampl
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
MIC_REPLAY_MAX_BYTES, build_wav_bytes, camera_preview_pipeline_desc,
|
MIC_REPLAY_MAX_BYTES, build_wav_bytes, camera_preview_mode, camera_preview_pipeline_desc,
|
||||||
microphone_monitor_pipeline_desc, normalize_camera_selection, push_recent_audio,
|
microphone_monitor_pipeline_desc, normalize_camera_selection, push_recent_audio,
|
||||||
read_camera_preview_tap, read_microphone_level_tap, resolve_camera_device,
|
read_camera_preview_tap, read_microphone_level_tap, resolve_camera_device,
|
||||||
};
|
};
|
||||||
@ -1152,6 +1192,17 @@ mod tests {
|
|||||||
desc.contains("image/jpeg,width=(int)1920,height=(int)1080,framerate=(fraction)30/1")
|
desc.contains("image/jpeg,width=(int)1920,height=(int)1080,framerate=(fraction)30/1")
|
||||||
);
|
);
|
||||||
assert!(desc.contains("decodebin ! videoconvert ! videoscale"));
|
assert!(desc.contains("decodebin ! videoconvert ! videoscale"));
|
||||||
|
assert!(desc.contains("video/x-raw,format=RGBA,width=1920,height=1080,framerate=30/1"));
|
||||||
|
assert!(!desc.contains("width=128,height=72"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn camera_preview_mode_defaults_to_hd_and_tracks_selected_quality() {
|
||||||
|
assert_eq!(camera_preview_mode(None), (1280, 720, 30));
|
||||||
|
assert_eq!(
|
||||||
|
camera_preview_mode(Some(CameraMode::new(1920, 1080, 30))),
|
||||||
|
(1920, 1080, 30)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@ -68,6 +68,21 @@ mod camera_include_contract {
|
|||||||
assert!(chain.contains("capsfilter caps=\""));
|
assert!(chain.contains("capsfilter caps=\""));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn camera_preview_tap_uses_the_actual_uplink_dimensions() {
|
||||||
|
let branch = camera_preview_tap_branch(1920, 1080, 30);
|
||||||
|
assert!(branch.contains("width=1920,height=1080"));
|
||||||
|
assert!(branch.contains("framerate=30/1"));
|
||||||
|
assert!(!branch.contains("width=128,height=72"));
|
||||||
|
|
||||||
|
let branch = camera_preview_tap_branch(1280, 720, 120);
|
||||||
|
assert!(branch.contains("width=1280,height=720"));
|
||||||
|
assert!(
|
||||||
|
branch.contains("framerate=60/1"),
|
||||||
|
"preview tap should cap frame production while preserving resolution"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn encoder_helpers_return_supported_defaults() {
|
fn encoder_helpers_return_supported_defaults() {
|
||||||
init_gst();
|
init_gst();
|
||||||
@ -202,6 +217,16 @@ mod camera_include_contract {
|
|||||||
bytes.starts_with(b"LESAVKA_RGBA "),
|
bytes.starts_with(b"LESAVKA_RGBA "),
|
||||||
"preview tap should publish an RGBA frame header"
|
"preview tap should publish an RGBA frame header"
|
||||||
);
|
);
|
||||||
|
let header_end = bytes
|
||||||
|
.iter()
|
||||||
|
.position(|byte| *byte == b'\n')
|
||||||
|
.expect("preview header newline");
|
||||||
|
let header =
|
||||||
|
std::str::from_utf8(&bytes[..header_end]).expect("utf8 header");
|
||||||
|
assert!(
|
||||||
|
header.starts_with("LESAVKA_RGBA 160 90 "),
|
||||||
|
"preview tap should prove the selected uplink size, got {header:?}"
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user