diff --git a/AGENTS.md b/AGENTS.md index aebb4e6..b95f12d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Lesavka Agent Notes -## 0.18.4 Bundled Webcam A/V Migration Checklist +## 0.18.5 Bundled Webcam A/V Migration Checklist Context: manual Google Meet and mirrored-probe testing showed the split webcam and microphone uplink design is too fragile under real browser/device pressure. @@ -31,9 +31,9 @@ explicit no-camera path. - [x] An already-attached UVC gadget descriptor is the physical browser contract: if it still advertises an older profile, server handshake/capture sizing follows that live descriptor until a controlled gadget rebuild is allowed. -- [x] Bundled webcam sessions enforce a hard one-second freshness ceiling: - server-side reanchors may improve smoothness, but stale/future waits over - the live budget are dropped instead of preserving lag. +- [x] Bundled webcam sessions enforce freshness before output-path compensation; + sync-critical measured UVC/UAC offsets are allowed to add presentation + delay because audio/video sync is higher priority than freshness. - [x] Bundled webcam sessions use the shared client capture timeline for transit sync, then apply runtime output-path calibration when splitting into UVC/UAC so Meet sees synchronized presentation. @@ -76,12 +76,11 @@ explicit no-camera path. raw packet `pts`, and reset the bundled epoch on client-session changes. - [x] Keep server freshness drops/reanchors active for bundled media. - [x] Drop mixed A/V bundles coherently when one side fails freshness/sync planning. -- [x] Reanchor bundled playout when due times drift too far into the future, and - drop packets whose predicted playout age would exceed the one-second budget. -- [x] Keep bundled audio scheduled early enough that UVC/UAC output-path - compensation can fit inside the one-second freshness budget when possible. -- [x] Trim bundled output-path offset spans only when the active calibration - would otherwise violate the one-second freshness ceiling. +- [x] Reanchor bundled playout when due times drift too far before output-path + compensation, and drop packets whose predicted pre-compensation playout + age would exceed the one-second freshness budget. +- [x] Keep bundled UVC/UAC output-path compensation authoritative; do not clip + measured offsets just to improve freshness when that would break sync. - [x] Activate the camera relay before opening the microphone sink so UVC can become ready even if UAC setup is slow. - [x] Log the first bundled video frame handed to the camera sink. diff --git a/Cargo.lock b/Cargo.lock index 639541e..6e6264b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.18.4" +version = "0.18.5" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.18.4" +version = "0.18.5" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.18.4" +version = "0.18.5" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 800d03c..29c7143 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.18.4" +version = "0.18.5" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 7647797..2f3bac6 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.18.4" +version = "0.18.5" edition = "2024" build = "build.rs" diff --git a/docs/operational-env.md b/docs/operational-env.md index 62660dc..803a017 100644 --- a/docs/operational-env.md +++ b/docs/operational-env.md @@ -249,9 +249,9 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UAC_SESSION_CLOCK_ALIGN` | server audio sink clock-alignment override; `0` is the host-validated default | | `LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US` | legacy/split server upstream playout override; shifts gadget-audio presentation relative to the shared playout epoch | | `LESAVKA_UPSTREAM_AUDIO_MASTER_WAIT_GRACE_MS` | server upstream sync override; how long video may wait past its nominal due time for UAC audio to reach the matching timestamp, defaults to `350` | -| `LESAVKA_UPSTREAM_BUNDLED_AUDIO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime audio output-path calibration when unset, with the final A/V offset span trimmed to fit the live budget | -| `LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS` | bundled webcam jitter buffer before output-path compensation; defaults to `20` so compensated video can stay under the live budget | -| `LESAVKA_UPSTREAM_BUNDLED_VIDEO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime video output-path calibration when unset, with the final A/V offset span trimmed to fit the live budget | +| `LESAVKA_UPSTREAM_BUNDLED_AUDIO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime audio output-path calibration when unset; sync-critical measured offsets are not clipped for freshness | +| `LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS` | bundled webcam jitter buffer before output-path compensation; defaults to `350` to protect smooth synced playout | +| `LESAVKA_UPSTREAM_BUNDLED_VIDEO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime video output-path calibration when unset; sync-critical measured offsets are not clipped for freshness | | `LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS` | server upstream planner freshness ceiling; planner-approved audio/video should not exceed this live lag budget, defaults to `1000` and is capped at `1000` | | `LESAVKA_UPSTREAM_PAIR_SLACK_US` | server upstream pairing override; how far video may diverge from the planned audio-master capture moment before the frame is held or dropped, defaults to `80000` | | `LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS` | server upstream pairing/synchronization target buffer; the server uses this shared buffer to pair webcam frames with matching gadget-mic audio before remote presentation, defaults to `350` | diff --git a/server/Cargo.toml b/server/Cargo.toml index 40ee752..e13362d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.18.4" +version = "0.18.5" edition = "2024" autobins = false diff --git a/server/src/main/relay_service.rs b/server/src/main/relay_service.rs index ce504e0..77bf54d 100644 --- a/server/src/main/relay_service.rs +++ b/server/src/main/relay_service.rs @@ -127,7 +127,7 @@ fn bundled_upstream_playout_delay() -> Duration { .ok() .and_then(|value| value.trim().parse::().ok()) .map(Duration::from_millis) - .unwrap_or_else(|| Duration::from_millis(20)) + .unwrap_or_else(|| Duration::from_millis(350)) } #[cfg(not(coverage))] diff --git a/server/src/upstream_media_runtime.rs b/server/src/upstream_media_runtime.rs index 3490d47..4e3b918 100644 --- a/server/src/upstream_media_runtime.rs +++ b/server/src/upstream_media_runtime.rs @@ -115,39 +115,8 @@ impl UpstreamMediaRuntime { .unwrap_or_else(|| self.playout_offset_us(kind)) } - fn bundled_playout_offsets_us(&self) -> (i64, i64) { - let mut camera_offset_us = self.raw_bundled_playout_offset_us(UpstreamMediaKind::Camera); - let mut microphone_offset_us = - self.raw_bundled_playout_offset_us(UpstreamMediaKind::Microphone); - let max_span_us = upstream_max_live_lag() - .saturating_sub(upstream_bundled_playout_delay()) - .as_micros() - .min(i64::MAX as u128) as i64; - let span_us = camera_offset_us.saturating_sub(microphone_offset_us); - if span_us > max_span_us { - camera_offset_us = microphone_offset_us.saturating_add(max_span_us); - } else if span_us < -max_span_us { - microphone_offset_us = camera_offset_us.saturating_add(max_span_us); - } - (camera_offset_us, microphone_offset_us) - } - fn bundled_playout_offset_us(&self, kind: UpstreamMediaKind) -> i64 { - let (camera_offset_us, microphone_offset_us) = self.bundled_playout_offsets_us(); - match kind { - UpstreamMediaKind::Camera => camera_offset_us, - UpstreamMediaKind::Microphone => microphone_offset_us, - } - } - - fn bundled_later_offset_reserve_us(&self, kind: UpstreamMediaKind) -> u64 { - let (camera_offset_us, microphone_offset_us) = self.bundled_playout_offsets_us(); - let slowest_offset_us = camera_offset_us.max(microphone_offset_us); - let kind_offset_us = match kind { - UpstreamMediaKind::Camera => camera_offset_us, - UpstreamMediaKind::Microphone => microphone_offset_us, - }; - slowest_offset_us.saturating_sub(kind_offset_us).max(0) as u64 + self.raw_bundled_playout_offset_us(kind) } /// Mark one audio chunk as actually handed to the UAC sink. @@ -432,12 +401,18 @@ impl UpstreamMediaRuntime { let playout_delay = upstream_bundled_playout_delay().min(max_live_lag); let reanchor_threshold = upstream_reanchor_late_threshold(playout_delay); let max_future_wait = max_live_lag.saturating_sub(source_lag); - let later_offset_reserve = - Duration::from_micros(self.bundled_later_offset_reserve_us(kind)); - let max_kind_future_wait = max_future_wait.saturating_sub(later_offset_reserve); + let output_offset = if sink_offset_us >= 0 { + Duration::from_micros(sink_offset_us as u64) + } else { + Duration::ZERO + }; let due_future_wait = due_at.saturating_duration_since(now); - if late_by > reanchor_threshold || due_future_wait > max_kind_future_wait { - let desired_delay = playout_delay.min(max_kind_future_wait); + let due_future_wait_before_output_compensation = + due_future_wait.saturating_sub(output_offset); + if late_by > reanchor_threshold + || due_future_wait_before_output_compensation > max_future_wait + { + let desired_delay = playout_delay.min(max_future_wait); let desired_due_at = now + desired_delay; let unoffset_due_at = apply_playout_offset(desired_due_at, -sink_offset_us); let recovered_epoch = unoffset_due_at @@ -462,31 +437,30 @@ impl UpstreamMediaRuntime { recovery_buffer_ms = desired_delay.as_millis(), max_live_lag_ms = max_live_lag.as_millis(), source_lag_ms = source_lag.as_millis(), - later_offset_reserve_ms = later_offset_reserve.as_millis(), + output_offset_ms = output_offset.as_millis(), "bundled upstream media playhead reanchored to preserve freshness" ); } - let predicted_lag_at_playout = - source_lag.saturating_add(due_at.saturating_duration_since(now)); - if predicted_lag_at_playout > max_live_lag { + let predicted_lag_before_output_compensation = source_lag.saturating_add( + due_at + .saturating_duration_since(now) + .saturating_sub(output_offset), + ); + if predicted_lag_before_output_compensation > max_live_lag { match kind { UpstreamMediaKind::Camera => { state.stale_video_drops = state.stale_video_drops.saturating_add(1); state.video_freezes = state.video_freezes.saturating_add(1); - state.last_reason = - "dropped bundled video that would exceed max live lag at playout" - .to_string(); + state.last_reason = "dropped bundled video that would exceed max live lag before output compensation".to_string(); } UpstreamMediaKind::Microphone => { state.stale_audio_drops = state.stale_audio_drops.saturating_add(1); - state.last_reason = - "dropped bundled audio that would exceed max live lag at playout" - .to_string(); + state.last_reason = "dropped bundled audio that would exceed max live lag before output compensation".to_string(); } } state.phase = UpstreamSyncPhase::Healing; return UpstreamPlanDecision::DropStale( - "bundled packet would exceed max live lag at playout", + "bundled packet would exceed max live lag before output compensation", ); } diff --git a/server/src/upstream_media_runtime/config.rs b/server/src/upstream_media_runtime/config.rs index 54bc175..c75eac2 100644 --- a/server/src/upstream_media_runtime/config.rs +++ b/server/src/upstream_media_runtime/config.rs @@ -30,7 +30,7 @@ pub(super) fn upstream_bundled_playout_delay() -> Duration { .or_else(|_| std::env::var("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS")) .ok() .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(20); + .unwrap_or(350); Duration::from_millis(delay_ms) } diff --git a/server/src/upstream_media_runtime/tests/config.rs b/server/src/upstream_media_runtime/tests/config.rs index 6aa7ad7..2673f17 100644 --- a/server/src/upstream_media_runtime/tests/config.rs +++ b/server/src/upstream_media_runtime/tests/config.rs @@ -17,12 +17,12 @@ fn upstream_playout_delay_defaults_to_freshness_budget_and_accepts_overrides() { #[test] #[serial(upstream_media_runtime)] -fn upstream_bundled_playout_delay_defaults_low_and_accepts_overrides() { +fn upstream_bundled_playout_delay_defaults_to_smooth_sync_buffer_and_accepts_overrides() { temp_env::with_var_unset("LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", || { temp_env::with_var_unset("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", || { assert_eq!( super::upstream_bundled_playout_delay(), - Duration::from_millis(20) + Duration::from_millis(350) ); }); }); diff --git a/server/src/upstream_media_runtime/tests/planning.rs b/server/src/upstream_media_runtime/tests/planning.rs index 0a04bde..6f460f1 100644 --- a/server/src/upstream_media_runtime/tests/planning.rs +++ b/server/src/upstream_media_runtime/tests/planning.rs @@ -167,7 +167,7 @@ fn bundled_media_explicit_offsets_can_disable_runtime_output_calibration() { #[test] #[serial(upstream_media_runtime)] -fn bundled_media_schedules_audio_early_so_compensated_video_stays_fresh() { +fn bundled_media_applies_output_compensation_without_clipping_for_freshness() { temp_env::with_var( "LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", Some("20"), @@ -214,7 +214,7 @@ fn bundled_media_schedules_audio_early_so_compensated_video_stays_fresh() { #[test] #[serial(upstream_media_runtime)] -fn bundled_media_clamps_output_compensation_to_freshness_budget() { +fn bundled_media_preserves_large_measured_output_compensation_for_sync() { temp_env::with_var( "LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", Some("20"), @@ -244,18 +244,64 @@ fn bundled_media_clamps_output_compensation_to_freshness_budget() { assert_eq!( video.due_at.saturating_duration_since(audio.due_at), - Duration::from_millis(980), - "factory output compensation should be trimmed only enough to respect freshness" + Duration::from_millis(1090), + "measured output compensation is sync-critical and must not be clipped" ); assert!( - video.due_at.saturating_duration_since(now) <= Duration::from_secs(1), - "clamped output compensation should preserve the live ceiling" + video.due_at.saturating_duration_since(now) > Duration::from_secs(1), + "sync wins over freshness when measured UVC/UAC egress latency requires it" ); }); }, ); } +#[test] +#[serial(upstream_media_runtime)] +fn bundled_media_keeps_default_smooth_buffer_and_measured_output_compensation() { + temp_env::with_var_unset("LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", || { + temp_env::with_var_unset("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", || { + temp_env::with_var("LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS", Some("1000"), || { + let runtime = UpstreamMediaRuntime::new(); + runtime.set_playout_offsets(1_090_000, 0); + let _camera = runtime.activate_camera(); + let _microphone = runtime.activate_microphone(); + let now = tokio::time::Instant::now(); + let epoch = now + Duration::from_millis(350); + + let audio = play(runtime.plan_bundled_pts( + super::UpstreamMediaKind::Microphone, + 1_000_000, + 1, + 1_000_000, + epoch, + )); + let video = play(runtime.plan_bundled_pts( + super::UpstreamMediaKind::Camera, + 1_000_000, + 16_666, + 1_000_000, + epoch, + )); + + assert!( + audio.due_at.saturating_duration_since(now) >= Duration::from_millis(300), + "the default bundled jitter buffer must remain available for smoothness" + ); + assert_eq!( + video.due_at.saturating_duration_since(audio.due_at), + Duration::from_millis(1090), + "measured UVC/UAC egress compensation remains authoritative after the buffer" + ); + assert!( + video.due_at.saturating_duration_since(now) > Duration::from_secs(1), + "sync-critical output delay may exceed the live budget because sync wins" + ); + }); + }); + }); +} + #[test] #[serial(upstream_media_runtime)] fn bundled_media_reanchors_future_wait_inside_one_second_freshness_budget() {