diff --git a/Cargo.lock b/Cargo.lock index 1ff606b..1779d6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.22.42" +version = "0.22.43" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.22.42" +version = "0.22.43" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.22.42" +version = "0.22.43" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 09accde..529c451 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.22.42" +version = "0.22.43" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index e19fa3a..b62d1f0 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.22.42" +version = "0.22.43" edition = "2024" build = "build.rs" diff --git a/docs/operational-env.md b/docs/operational-env.md index 842bb16..03f7c44 100644 --- a/docs/operational-env.md +++ b/docs/operational-env.md @@ -297,6 +297,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS` | server upstream startup guard; paired startup must converge before this timeout or fail visibly, defaults to `60000` | | `LESAVKA_UPSTREAM_STALE_DROP_MS` | server upstream freshness override; late audio/video that miss this budget are dropped instead of silently extending lag, defaults to `80` | | `LESAVKA_UPSTREAM_TIMING_TRACE` | upstream capture/rebase trace override for sync debugging | +| `LESAVKA_UPSTREAM_V2_AUDIO_STARVATION_HEAL_MS` | v2 bundled webcam watchdog; if video stays live while audio disappears for this long, the server retires the mic epoch and forces a clean bundled reconnect, defaults to `1500`, set `0` to disable | | `LESAVKA_UPSTREAM_V2_MAX_LIVE_AGE_MS` | v2 bundled webcam freshness ceiling; bundles already older than this are dropped as one unit, defaults to `1000` | | `LESAVKA_UPSTREAM_V2_PLAYOUT_DELAY_MS` | v2 optional common playout slack after sync offsets; defaults to `20` and is reduced when needed to protect the live-age budget | | `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_MODE_OFFSETS_US` | server upstream per-UVC-mode output-path map; shipped MJPEG defaults are `1280x720@20=162659,1280x720@30=135090,1920x1080@20=160045,1920x1080@30=127952`; shipped HEVC decode-to-MJPEG defaults are profile-specific under `LESAVKA_UPSTREAM_HEVC_VIDEO_PLAYOUT_MODE_OFFSETS_US` | @@ -314,7 +315,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UVC_APP_MAX_TIME_NS` | server UVC appsrc memory guard; defaults to `200000000` ns of queued media | | `LESAVKA_UVC_BLOCKING` | server hardware/device override | | `LESAVKA_UVC_BULK` | UVC transfer-mode override; defaults to `1` so patched kernels prefer reliable bulk transfer over lossy isochronous MJPEG. If the live kernel does not expose `streaming_bulk`, Lesavka falls back to isochronous descriptors without the 512-byte bulk clamp. Set `0` to force classic isochronous descriptors | -| `LESAVKA_UVC_BUFFER_COUNT` | UVC helper freshness override; number of queued gadget output buffers, defaults to `2` for live-call freshness | +| `LESAVKA_UVC_BUFFER_COUNT` | UVC helper freshness override; number of queued gadget output buffers, defaults to `4` to keep the gadget fed through host/browser scheduling jitter | | `LESAVKA_UVC_BY_PATH_ROOT` | server hardware/device override | | `LESAVKA_UVC_CODEC` | server hardware/device override | | `LESAVKA_UVC_CTRL_BIN` | server hardware/device override | @@ -345,6 +346,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UVC_MAXPAYLOAD_LIMIT` | server hardware/device override | | `LESAVKA_UVC_MJPEG` | server hardware/device override | | `LESAVKA_UVC_MJPEG_BUDGET_BYTES_PER_SEC` | UVC helper MJPEG budget guard; derives a per-frame byte cap from target FPS when `LESAVKA_UVC_FRAME_MAX_BYTES` is unset; non-bulk UVC is additionally capped by `LESAVKA_UVC_ISOCHRONOUS_LIMIT_PCT` | +| `LESAVKA_UVC_QUEUE_PACING` | UVC helper queue pacing override; defaults to `0` because the RCT host already paces UVC consumption, and delaying returned buffer requeueing can starve isochronous gadget transfers | | `LESAVKA_UVC_SKIP_UDEV` | server hardware/device override | | `LESAVKA_UVC_STATS_INTERVAL_MS` | UVC helper telemetry interval for queued/reloaded/rejected MJPEG frame counters; defaults to `5000`, `0` disables | | `LESAVKA_UVC_STATS_PATH` | UVC helper JSON stats snapshot path for queued/reloaded/rejected MJPEG frame counters; defaults to `/run/lesavka-uvc-video-stats.json`, set `0` or empty to disable file snapshots | diff --git a/scripts/install/server.sh b/scripts/install/server.sh index e014d3e..0833252 100755 --- a/scripts/install/server.sh +++ b/scripts/install/server.sh @@ -355,7 +355,7 @@ LESAVKA_UVC_HEIGHT=$(uvc_env_value LESAVKA_UVC_HEIGHT 720) LESAVKA_UVC_CODEC=${INSTALL_UVC_CODEC} LESAVKA_UVC_BLOCKING=$(uvc_env_value LESAVKA_UVC_BLOCKING 1) LESAVKA_UVC_CONTROL_READ_ONLY=$(uvc_env_value LESAVKA_UVC_CONTROL_READ_ONLY 0) -LESAVKA_UVC_QUEUE_PACING=$(uvc_env_value LESAVKA_UVC_QUEUE_PACING 1) +LESAVKA_UVC_QUEUE_PACING=$(uvc_env_value LESAVKA_UVC_QUEUE_PACING 0) LESAVKA_UVC_MAXBURST=$(uvc_env_value LESAVKA_UVC_MAXBURST 0) LESAVKA_UVC_BULK=$(uvc_env_value LESAVKA_UVC_BULK 1) LESAVKA_UVC_FRAME_SIZE_GUARD=$(uvc_env_value LESAVKA_UVC_FRAME_SIZE_GUARD 1) diff --git a/server/Cargo.toml b/server/Cargo.toml index e0be19a..290fce6 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.22.42" +version = "0.22.43" edition = "2024" autobins = false diff --git a/server/src/bin/lesavka-uvc.real.inc b/server/src/bin/lesavka-uvc.real.inc index 1148f77..d893d66 100644 --- a/server/src/bin/lesavka-uvc.real.inc +++ b/server/src/bin/lesavka-uvc.real.inc @@ -48,7 +48,7 @@ const V4L2_PIX_FMT_MJPEG: u32 = u32::from_le_bytes(*b"MJPG"); const MAX_MJPEG_FRAME_BYTES: usize = 8 * 1024 * 1024; const MINIMAL_MJPEG_FRAME: &[u8] = &[0xff, 0xd8, 0xff, 0xd9]; const IDLE_MJPEG_FRAME: &[u8] = include_bytes!("lesavka_uvc/idle_1280x720_black.jpg"); -const DEFAULT_UVC_BUFFER_COUNT: u32 = 2; +const DEFAULT_UVC_BUFFER_COUNT: u32 = 4; const DEFAULT_UVC_IDLE_PUMP_MS: u64 = 2; const DEFAULT_UVC_FRAME_MAX_AGE_MS: u64 = 1_000; const DEFAULT_UVC_MJPEG_BUDGET_BYTES_PER_SEC: u32 = 10_000_000; @@ -657,7 +657,7 @@ fn uvc_idle_pump_sleep() -> Duration { } fn uvc_queue_period(fps: u32) -> Option { - if !env_flag_enabled("LESAVKA_UVC_QUEUE_PACING", true) { + if !env_flag_enabled("LESAVKA_UVC_QUEUE_PACING", false) { return None; } let fps = fps.max(1); diff --git a/server/src/bin/lesavka_uvc/coverage_model.rs b/server/src/bin/lesavka_uvc/coverage_model.rs index 833acb9..0c76fab 100644 --- a/server/src/bin/lesavka_uvc/coverage_model.rs +++ b/server/src/bin/lesavka_uvc/coverage_model.rs @@ -50,7 +50,7 @@ const UVC_VS_COMMIT_CONTROL: u8 = 0x02; const UVC_VC_REQUEST_ERROR_CODE_CONTROL: u8 = 0x02; #[cfg(coverage)] -const DEFAULT_UVC_BUFFER_COUNT: u32 = 2; +const DEFAULT_UVC_BUFFER_COUNT: u32 = 4; #[cfg(coverage)] const DEFAULT_UVC_IDLE_PUMP_MS: u64 = 2; #[cfg(coverage)] diff --git a/server/src/bin/lesavka_uvc/coverage_startup.rs b/server/src/bin/lesavka_uvc/coverage_startup.rs index f9ecd70..ce86eff 100644 --- a/server/src/bin/lesavka_uvc/coverage_startup.rs +++ b/server/src/bin/lesavka_uvc/coverage_startup.rs @@ -159,7 +159,7 @@ fn uvc_idle_pump_sleep() -> std::time::Duration { /// `None` when pacing is explicitly disabled. Why: the RCT-facing host must /// not be overfed faster than the advertised descriptor cadence. fn uvc_queue_period(fps: u32) -> Option { - if !env_flag_enabled("LESAVKA_UVC_QUEUE_PACING", true) { + if !env_flag_enabled("LESAVKA_UVC_QUEUE_PACING", false) { return None; } let fps = fps.max(1); diff --git a/server/src/bin/tests/lesavka_uvc.rs b/server/src/bin/tests/lesavka_uvc.rs index 02a10d0..e66e619 100644 --- a/server/src/bin/tests/lesavka_uvc.rs +++ b/server/src/bin/tests/lesavka_uvc.rs @@ -135,6 +135,20 @@ fn uvc_frame_budget_caps_isochronous_transport() { ); } +#[test] +fn uvc_queue_pacing_defaults_off_but_remains_opt_in() { + with_var("LESAVKA_UVC_QUEUE_PACING", None::<&str>, || { + assert_eq!(uvc_queue_period(30), None); + }); + + with_var("LESAVKA_UVC_QUEUE_PACING", Some("1"), || { + assert_eq!( + uvc_queue_period(30), + Some(std::time::Duration::from_nanos(33_333_333)) + ); + }); +} + #[test] #[serial] fn main_coverage_mode_returns_error_for_non_uvc_node() { diff --git a/server/src/main/relay_service.rs b/server/src/main/relay_service.rs index 449f8cd..2c1fdab 100644 --- a/server/src/main/relay_service.rs +++ b/server/src/main/relay_service.rs @@ -2,6 +2,7 @@ const MEDIA_V2_DEFAULT_PLAYOUT_DELAY_MS: u64 = 20; const MEDIA_V2_DEFAULT_MAX_LIVE_AGE_MS: u64 = 1_000; const MEDIA_V2_DEFAULT_UAC_START_TIMEOUT_MS: u64 = 750; const MEDIA_V2_DEFAULT_STREAM_IDLE_TIMEOUT_MS: u64 = 5_000; +const MEDIA_V2_DEFAULT_AUDIO_STARVATION_HEAL_MS: u64 = 1_500; const MEDIA_V2_MAX_MIXED_CAPTURE_SPAN_US: u64 = 250_000; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -126,6 +127,39 @@ fn media_v2_stream_idle_timeout() -> Duration { .unwrap_or_else(|| Duration::from_millis(MEDIA_V2_DEFAULT_STREAM_IDLE_TIMEOUT_MS)) } +fn media_v2_audio_starvation_heal_after() -> Option { + let millis = std::env::var("LESAVKA_UPSTREAM_V2_AUDIO_STARVATION_HEAL_MS") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(MEDIA_V2_DEFAULT_AUDIO_STARVATION_HEAL_MS); + (millis > 0).then(|| Duration::from_millis(millis)) +} + +/// Track whether a bundled stream has lost audio while video is still live. +/// +/// Inputs are the latest bundle facts plus the current starvation marker. +/// Output is true once the audio-free video run exceeds the heal window. Why: +/// a stuck mic capture path can leave the RCT hearing a stale UAC epoch while +/// fresh video keeps flowing; retiring the mic generation forces the client to +/// rebuild the bundled stream without touching USB descriptors. +fn media_v2_audio_starvation_elapsed( + audio_enabled: bool, + facts: MediaV2BundleFacts, + now: tokio::time::Instant, + starved_since: &mut Option, + heal_after: Option, +) -> bool { + if !audio_enabled || !facts.has_video || facts.has_audio { + *starved_since = None; + return false; + } + let Some(heal_after) = heal_after else { + return false; + }; + let since = starved_since.get_or_insert(now); + now.saturating_duration_since(*since) >= heal_after +} + /// Keeps `media_v2_handoff_schedule` explicit because it sits on relay RPC orchestration, where hardware failures must surface without stopping the server. /// Inputs are the typed parameters; output is the return value or side effect. fn media_v2_handoff_schedule( diff --git a/server/src/main/relay_service/upstream_media_rpc.rs b/server/src/main/relay_service/upstream_media_rpc.rs index 0d49537..2d5ed60 100644 --- a/server/src/main/relay_service/upstream_media_rpc.rs +++ b/server/src/main/relay_service/upstream_media_rpc.rs @@ -84,6 +84,8 @@ impl Handler { let mut waiting_for_hevc_keyframe = false; let mut outcome = "aborted"; let idle_timeout = media_v2_stream_idle_timeout(); + let audio_starvation_heal_after = media_v2_audio_starvation_heal_after(); + let mut audio_starved_since = None; let (mut audio_handoff_tx, audio_worker) = if let Some((microphone_sink_permit, sink)) = microphone_sink { let (audio_handoff_tx, audio_handoff_rx) = @@ -174,6 +176,29 @@ impl Handler { let Some(facts) = summarize_media_v2_bundle(&bundle) else { continue; }; + if media_v2_audio_starvation_elapsed( + audio_enabled, + facts, + bundle_arrived_at, + &mut audio_starved_since, + audio_starvation_heal_after, + ) { + let audio_starved_ms = audio_starved_since + .map(|since| bundle_arrived_at.saturating_duration_since(since)) + .unwrap_or_default() + .as_millis(); + warn!( + rpc_id, + session_id = camera_lease.session_id, + client_bundle_session_id = bundle.session_id, + bundle_seq = bundle.seq, + audio_starved_ms, + "📦🩺 v2 bundled microphone starved while video stayed live; retiring upstream audio epoch" + ); + upstream_media_rt.soft_recover_microphone(); + outcome = "audio-starved"; + break; + } if facts.has_audio && facts.has_video && facts.capture_span_us > MEDIA_V2_MAX_MIXED_CAPTURE_SPAN_US { if matches!(camera_cfg.codec, camera::CameraCodec::Hevc) && facts.has_video { waiting_for_hevc_keyframe = true; diff --git a/server/src/main/relay_service_tests.rs b/server/src/main/relay_service_tests.rs index eacc3e9..0eb22cf 100644 --- a/server/src/main/relay_service_tests.rs +++ b/server/src/main/relay_service_tests.rs @@ -2,11 +2,11 @@ #[allow(clippy::items_after_test_module)] mod tests { use super::{ - MediaV2BundleFacts, UpstreamStreamCleanup, media_v2_frame_step_us, - media_v2_handoff_schedule, media_v2_has_hevc_recovery_keyframe, - media_v2_should_hold_hevc_video_for_recovery, prepare_media_v2_audio, - prepare_media_v2_video, retain_freshest_audio_packet, retain_freshest_video_packet, - summarize_media_v2_bundle, media_v2_uac_start_timeout, + MediaV2BundleFacts, UpstreamStreamCleanup, media_v2_audio_starvation_elapsed, + media_v2_audio_starvation_heal_after, media_v2_frame_step_us, media_v2_handoff_schedule, + media_v2_has_hevc_recovery_keyframe, media_v2_should_hold_hevc_video_for_recovery, + media_v2_uac_start_timeout, prepare_media_v2_audio, prepare_media_v2_video, + retain_freshest_audio_packet, retain_freshest_video_packet, summarize_media_v2_bundle, }; use lesavka_common::lesavka::{AudioPacket, UpstreamMediaBundle, VideoPacket}; use lesavka_server::camera::CameraCodec; @@ -196,6 +196,74 @@ mod tests { }); } + #[test] + fn media_v2_audio_starvation_watchdog_is_short_and_disableable() { + temp_env::with_var_unset("LESAVKA_UPSTREAM_V2_AUDIO_STARVATION_HEAL_MS", || { + assert_eq!( + media_v2_audio_starvation_heal_after(), + Some(std::time::Duration::from_millis(1_500)) + ); + }); + + temp_env::with_var( + "LESAVKA_UPSTREAM_V2_AUDIO_STARVATION_HEAL_MS", + Some("0"), + || { + assert_eq!(media_v2_audio_starvation_heal_after(), None); + }, + ); + } + + #[test] + fn media_v2_audio_starvation_elapsed_requires_live_video_without_audio() { + let mut starved_since = None; + let now = tokio::time::Instant::now(); + let video_only = MediaV2BundleFacts { + has_audio: false, + has_video: true, + capture_start_us: 1_000, + capture_end_us: 1_000, + capture_span_us: 0, + max_queue_age_ms: 0, + }; + let audio_video = MediaV2BundleFacts { + has_audio: true, + has_video: true, + ..video_only + }; + + assert!(!media_v2_audio_starvation_elapsed( + true, + video_only, + now, + &mut starved_since, + Some(std::time::Duration::from_millis(1_500)) + )); + assert!(starved_since.is_some()); + assert!(!media_v2_audio_starvation_elapsed( + true, + video_only, + now + std::time::Duration::from_millis(1_499), + &mut starved_since, + Some(std::time::Duration::from_millis(1_500)) + )); + assert!(media_v2_audio_starvation_elapsed( + true, + video_only, + now + std::time::Duration::from_millis(1_500), + &mut starved_since, + Some(std::time::Duration::from_millis(1_500)) + )); + assert!(!media_v2_audio_starvation_elapsed( + true, + audio_video, + now + std::time::Duration::from_millis(1_600), + &mut starved_since, + Some(std::time::Duration::from_millis(1_500)) + )); + assert!(starved_since.is_none()); + } + #[test] /// Keeps server HEVC drop recovery explicit because late-drop freshness can otherwise corrupt decoded video. fn media_v2_hevc_recovery_holds_delta_until_keyframe() { diff --git a/tests/contract/server/uvc/server_uvc_binary_extra_contract.rs b/tests/contract/server/uvc/server_uvc_binary_extra_contract.rs index bf9f7b6..767ef42 100644 --- a/tests/contract/server/uvc/server_uvc_binary_extra_contract.rs +++ b/tests/contract/server/uvc/server_uvc_binary_extra_contract.rs @@ -359,12 +359,9 @@ mod uvc_binary_extra { with_var("LESAVKA_UVC_BUFFER_COUNT", None::<&str>, || { with_var("LESAVKA_UVC_IDLE_PUMP_MS", None::<&str>, || { with_var("LESAVKA_UVC_FRAME_MAX_AGE_MS", None::<&str>, || { - assert_eq!(uvc_buffer_count(), 2); + assert_eq!(uvc_buffer_count(), 4); assert_eq!(uvc_idle_pump_sleep(), std::time::Duration::from_millis(2)); - assert_eq!( - uvc_queue_period(30), - Some(std::time::Duration::from_nanos(33_333_333)) - ); + assert_eq!(uvc_queue_period(30), None); assert_eq!( frame_spool_max_age(), Some(std::time::Duration::from_millis(1_000)) @@ -380,10 +377,13 @@ mod uvc_binary_extra { with_var("LESAVKA_UVC_BUFFER_COUNT", Some("99"), || { with_var("LESAVKA_UVC_IDLE_PUMP_MS", Some("11"), || { with_var("LESAVKA_UVC_FRAME_MAX_AGE_MS", Some("0"), || { - with_var("LESAVKA_UVC_QUEUE_PACING", Some("0"), || { + with_var("LESAVKA_UVC_QUEUE_PACING", Some("1"), || { assert_eq!(uvc_buffer_count(), 8); assert_eq!(uvc_idle_pump_sleep(), std::time::Duration::from_millis(11)); - assert_eq!(uvc_queue_period(30), None); + assert_eq!( + uvc_queue_period(30), + Some(std::time::Duration::from_nanos(33_333_333)) + ); assert_eq!(frame_spool_max_age(), None); }); });