fix: keep bundled egress sync authoritative

This commit is contained in:
Brad Stein 2026-05-03 11:26:54 -03:00
parent c1a3205a7c
commit 248f1b7a47
11 changed files with 96 additions and 77 deletions

View File

@ -1,6 +1,6 @@
# Lesavka Agent Notes # Lesavka Agent Notes
## 0.18.4 Bundled Webcam A/V Migration Checklist ## 0.18.5 Bundled Webcam A/V Migration Checklist
Context: manual Google Meet and mirrored-probe testing showed the split webcam Context: manual Google Meet and mirrored-probe testing showed the split webcam
and microphone uplink design is too fragile under real browser/device pressure. and microphone uplink design is too fragile under real browser/device pressure.
@ -31,9 +31,9 @@ explicit no-camera path.
- [x] An already-attached UVC gadget descriptor is the physical browser contract: - [x] An already-attached UVC gadget descriptor is the physical browser contract:
if it still advertises an older profile, server handshake/capture sizing if it still advertises an older profile, server handshake/capture sizing
follows that live descriptor until a controlled gadget rebuild is allowed. follows that live descriptor until a controlled gadget rebuild is allowed.
- [x] Bundled webcam sessions enforce a hard one-second freshness ceiling: - [x] Bundled webcam sessions enforce freshness before output-path compensation;
server-side reanchors may improve smoothness, but stale/future waits over sync-critical measured UVC/UAC offsets are allowed to add presentation
the live budget are dropped instead of preserving lag. delay because audio/video sync is higher priority than freshness.
- [x] Bundled webcam sessions use the shared client capture timeline for - [x] Bundled webcam sessions use the shared client capture timeline for
transit sync, then apply runtime output-path calibration when splitting transit sync, then apply runtime output-path calibration when splitting
into UVC/UAC so Meet sees synchronized presentation. into UVC/UAC so Meet sees synchronized presentation.
@ -76,12 +76,11 @@ explicit no-camera path.
raw packet `pts`, and reset the bundled epoch on client-session changes. raw packet `pts`, and reset the bundled epoch on client-session changes.
- [x] Keep server freshness drops/reanchors active for bundled media. - [x] Keep server freshness drops/reanchors active for bundled media.
- [x] Drop mixed A/V bundles coherently when one side fails freshness/sync planning. - [x] Drop mixed A/V bundles coherently when one side fails freshness/sync planning.
- [x] Reanchor bundled playout when due times drift too far into the future, and - [x] Reanchor bundled playout when due times drift too far before output-path
drop packets whose predicted playout age would exceed the one-second budget. compensation, and drop packets whose predicted pre-compensation playout
- [x] Keep bundled audio scheduled early enough that UVC/UAC output-path age would exceed the one-second freshness budget.
compensation can fit inside the one-second freshness budget when possible. - [x] Keep bundled UVC/UAC output-path compensation authoritative; do not clip
- [x] Trim bundled output-path offset spans only when the active calibration measured offsets just to improve freshness when that would break sync.
would otherwise violate the one-second freshness ceiling.
- [x] Activate the camera relay before opening the microphone sink so UVC can - [x] Activate the camera relay before opening the microphone sink so UVC can
become ready even if UAC setup is slow. become ready even if UAC setup is slow.
- [x] Log the first bundled video frame handed to the camera sink. - [x] Log the first bundled video frame handed to the camera sink.

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "lesavka_client" name = "lesavka_client"
version = "0.18.4" version = "0.18.5"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_common" name = "lesavka_common"
version = "0.18.4" version = "0.18.5"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_server" name = "lesavka_server"
version = "0.18.4" version = "0.18.5"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",

View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package] [package]
name = "lesavka_client" name = "lesavka_client"
version = "0.18.4" version = "0.18.5"
edition = "2024" edition = "2024"
[dependencies] [dependencies]

View File

@ -1,6 +1,6 @@
[package] [package]
name = "lesavka_common" name = "lesavka_common"
version = "0.18.4" version = "0.18.5"
edition = "2024" edition = "2024"
build = "build.rs" build = "build.rs"

View File

@ -249,9 +249,9 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta
| `LESAVKA_UAC_SESSION_CLOCK_ALIGN` | server audio sink clock-alignment override; `0` is the host-validated default | | `LESAVKA_UAC_SESSION_CLOCK_ALIGN` | server audio sink clock-alignment override; `0` is the host-validated default |
| `LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US` | legacy/split server upstream playout override; shifts gadget-audio presentation relative to the shared playout epoch | | `LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US` | legacy/split server upstream playout override; shifts gadget-audio presentation relative to the shared playout epoch |
| `LESAVKA_UPSTREAM_AUDIO_MASTER_WAIT_GRACE_MS` | server upstream sync override; how long video may wait past its nominal due time for UAC audio to reach the matching timestamp, defaults to `350` | | `LESAVKA_UPSTREAM_AUDIO_MASTER_WAIT_GRACE_MS` | server upstream sync override; how long video may wait past its nominal due time for UAC audio to reach the matching timestamp, defaults to `350` |
| `LESAVKA_UPSTREAM_BUNDLED_AUDIO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime audio output-path calibration when unset, with the final A/V offset span trimmed to fit the live budget | | `LESAVKA_UPSTREAM_BUNDLED_AUDIO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime audio output-path calibration when unset; sync-critical measured offsets are not clipped for freshness |
| `LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS` | bundled webcam jitter buffer before output-path compensation; defaults to `20` so compensated video can stay under the live budget | | `LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS` | bundled webcam jitter buffer before output-path compensation; defaults to `350` to protect smooth synced playout |
| `LESAVKA_UPSTREAM_BUNDLED_VIDEO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime video output-path calibration when unset, with the final A/V offset span trimmed to fit the live budget | | `LESAVKA_UPSTREAM_BUNDLED_VIDEO_PLAYOUT_OFFSET_US` | bundled webcam server playout override; defaults to the active runtime video output-path calibration when unset; sync-critical measured offsets are not clipped for freshness |
| `LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS` | server upstream planner freshness ceiling; planner-approved audio/video should not exceed this live lag budget, defaults to `1000` and is capped at `1000` | | `LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS` | server upstream planner freshness ceiling; planner-approved audio/video should not exceed this live lag budget, defaults to `1000` and is capped at `1000` |
| `LESAVKA_UPSTREAM_PAIR_SLACK_US` | server upstream pairing override; how far video may diverge from the planned audio-master capture moment before the frame is held or dropped, defaults to `80000` | | `LESAVKA_UPSTREAM_PAIR_SLACK_US` | server upstream pairing override; how far video may diverge from the planned audio-master capture moment before the frame is held or dropped, defaults to `80000` |
| `LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS` | server upstream pairing/synchronization target buffer; the server uses this shared buffer to pair webcam frames with matching gadget-mic audio before remote presentation, defaults to `350` | | `LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS` | server upstream pairing/synchronization target buffer; the server uses this shared buffer to pair webcam frames with matching gadget-mic audio before remote presentation, defaults to `350` |

View File

@ -10,7 +10,7 @@ bench = false
[package] [package]
name = "lesavka_server" name = "lesavka_server"
version = "0.18.4" version = "0.18.5"
edition = "2024" edition = "2024"
autobins = false autobins = false

View File

@ -127,7 +127,7 @@ fn bundled_upstream_playout_delay() -> Duration {
.ok() .ok()
.and_then(|value| value.trim().parse::<u64>().ok()) .and_then(|value| value.trim().parse::<u64>().ok())
.map(Duration::from_millis) .map(Duration::from_millis)
.unwrap_or_else(|| Duration::from_millis(20)) .unwrap_or_else(|| Duration::from_millis(350))
} }
#[cfg(not(coverage))] #[cfg(not(coverage))]

View File

@ -115,39 +115,8 @@ impl UpstreamMediaRuntime {
.unwrap_or_else(|| self.playout_offset_us(kind)) .unwrap_or_else(|| self.playout_offset_us(kind))
} }
fn bundled_playout_offsets_us(&self) -> (i64, i64) {
let mut camera_offset_us = self.raw_bundled_playout_offset_us(UpstreamMediaKind::Camera);
let mut microphone_offset_us =
self.raw_bundled_playout_offset_us(UpstreamMediaKind::Microphone);
let max_span_us = upstream_max_live_lag()
.saturating_sub(upstream_bundled_playout_delay())
.as_micros()
.min(i64::MAX as u128) as i64;
let span_us = camera_offset_us.saturating_sub(microphone_offset_us);
if span_us > max_span_us {
camera_offset_us = microphone_offset_us.saturating_add(max_span_us);
} else if span_us < -max_span_us {
microphone_offset_us = camera_offset_us.saturating_add(max_span_us);
}
(camera_offset_us, microphone_offset_us)
}
fn bundled_playout_offset_us(&self, kind: UpstreamMediaKind) -> i64 { fn bundled_playout_offset_us(&self, kind: UpstreamMediaKind) -> i64 {
let (camera_offset_us, microphone_offset_us) = self.bundled_playout_offsets_us(); self.raw_bundled_playout_offset_us(kind)
match kind {
UpstreamMediaKind::Camera => camera_offset_us,
UpstreamMediaKind::Microphone => microphone_offset_us,
}
}
fn bundled_later_offset_reserve_us(&self, kind: UpstreamMediaKind) -> u64 {
let (camera_offset_us, microphone_offset_us) = self.bundled_playout_offsets_us();
let slowest_offset_us = camera_offset_us.max(microphone_offset_us);
let kind_offset_us = match kind {
UpstreamMediaKind::Camera => camera_offset_us,
UpstreamMediaKind::Microphone => microphone_offset_us,
};
slowest_offset_us.saturating_sub(kind_offset_us).max(0) as u64
} }
/// Mark one audio chunk as actually handed to the UAC sink. /// Mark one audio chunk as actually handed to the UAC sink.
@ -432,12 +401,18 @@ impl UpstreamMediaRuntime {
let playout_delay = upstream_bundled_playout_delay().min(max_live_lag); let playout_delay = upstream_bundled_playout_delay().min(max_live_lag);
let reanchor_threshold = upstream_reanchor_late_threshold(playout_delay); let reanchor_threshold = upstream_reanchor_late_threshold(playout_delay);
let max_future_wait = max_live_lag.saturating_sub(source_lag); let max_future_wait = max_live_lag.saturating_sub(source_lag);
let later_offset_reserve = let output_offset = if sink_offset_us >= 0 {
Duration::from_micros(self.bundled_later_offset_reserve_us(kind)); Duration::from_micros(sink_offset_us as u64)
let max_kind_future_wait = max_future_wait.saturating_sub(later_offset_reserve); } else {
Duration::ZERO
};
let due_future_wait = due_at.saturating_duration_since(now); let due_future_wait = due_at.saturating_duration_since(now);
if late_by > reanchor_threshold || due_future_wait > max_kind_future_wait { let due_future_wait_before_output_compensation =
let desired_delay = playout_delay.min(max_kind_future_wait); due_future_wait.saturating_sub(output_offset);
if late_by > reanchor_threshold
|| due_future_wait_before_output_compensation > max_future_wait
{
let desired_delay = playout_delay.min(max_future_wait);
let desired_due_at = now + desired_delay; let desired_due_at = now + desired_delay;
let unoffset_due_at = apply_playout_offset(desired_due_at, -sink_offset_us); let unoffset_due_at = apply_playout_offset(desired_due_at, -sink_offset_us);
let recovered_epoch = unoffset_due_at let recovered_epoch = unoffset_due_at
@ -462,31 +437,30 @@ impl UpstreamMediaRuntime {
recovery_buffer_ms = desired_delay.as_millis(), recovery_buffer_ms = desired_delay.as_millis(),
max_live_lag_ms = max_live_lag.as_millis(), max_live_lag_ms = max_live_lag.as_millis(),
source_lag_ms = source_lag.as_millis(), source_lag_ms = source_lag.as_millis(),
later_offset_reserve_ms = later_offset_reserve.as_millis(), output_offset_ms = output_offset.as_millis(),
"bundled upstream media playhead reanchored to preserve freshness" "bundled upstream media playhead reanchored to preserve freshness"
); );
} }
let predicted_lag_at_playout = let predicted_lag_before_output_compensation = source_lag.saturating_add(
source_lag.saturating_add(due_at.saturating_duration_since(now)); due_at
if predicted_lag_at_playout > max_live_lag { .saturating_duration_since(now)
.saturating_sub(output_offset),
);
if predicted_lag_before_output_compensation > max_live_lag {
match kind { match kind {
UpstreamMediaKind::Camera => { UpstreamMediaKind::Camera => {
state.stale_video_drops = state.stale_video_drops.saturating_add(1); state.stale_video_drops = state.stale_video_drops.saturating_add(1);
state.video_freezes = state.video_freezes.saturating_add(1); state.video_freezes = state.video_freezes.saturating_add(1);
state.last_reason = state.last_reason = "dropped bundled video that would exceed max live lag before output compensation".to_string();
"dropped bundled video that would exceed max live lag at playout"
.to_string();
} }
UpstreamMediaKind::Microphone => { UpstreamMediaKind::Microphone => {
state.stale_audio_drops = state.stale_audio_drops.saturating_add(1); state.stale_audio_drops = state.stale_audio_drops.saturating_add(1);
state.last_reason = state.last_reason = "dropped bundled audio that would exceed max live lag before output compensation".to_string();
"dropped bundled audio that would exceed max live lag at playout"
.to_string();
} }
} }
state.phase = UpstreamSyncPhase::Healing; state.phase = UpstreamSyncPhase::Healing;
return UpstreamPlanDecision::DropStale( return UpstreamPlanDecision::DropStale(
"bundled packet would exceed max live lag at playout", "bundled packet would exceed max live lag before output compensation",
); );
} }

View File

@ -30,7 +30,7 @@ pub(super) fn upstream_bundled_playout_delay() -> Duration {
.or_else(|_| std::env::var("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS")) .or_else(|_| std::env::var("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS"))
.ok() .ok()
.and_then(|value| value.trim().parse::<u64>().ok()) .and_then(|value| value.trim().parse::<u64>().ok())
.unwrap_or(20); .unwrap_or(350);
Duration::from_millis(delay_ms) Duration::from_millis(delay_ms)
} }

View File

@ -17,12 +17,12 @@ fn upstream_playout_delay_defaults_to_freshness_budget_and_accepts_overrides() {
#[test] #[test]
#[serial(upstream_media_runtime)] #[serial(upstream_media_runtime)]
fn upstream_bundled_playout_delay_defaults_low_and_accepts_overrides() { fn upstream_bundled_playout_delay_defaults_to_smooth_sync_buffer_and_accepts_overrides() {
temp_env::with_var_unset("LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", || { temp_env::with_var_unset("LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", || {
temp_env::with_var_unset("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", || { temp_env::with_var_unset("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", || {
assert_eq!( assert_eq!(
super::upstream_bundled_playout_delay(), super::upstream_bundled_playout_delay(),
Duration::from_millis(20) Duration::from_millis(350)
); );
}); });
}); });

View File

@ -167,7 +167,7 @@ fn bundled_media_explicit_offsets_can_disable_runtime_output_calibration() {
#[test] #[test]
#[serial(upstream_media_runtime)] #[serial(upstream_media_runtime)]
fn bundled_media_schedules_audio_early_so_compensated_video_stays_fresh() { fn bundled_media_applies_output_compensation_without_clipping_for_freshness() {
temp_env::with_var( temp_env::with_var(
"LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", "LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS",
Some("20"), Some("20"),
@ -214,7 +214,7 @@ fn bundled_media_schedules_audio_early_so_compensated_video_stays_fresh() {
#[test] #[test]
#[serial(upstream_media_runtime)] #[serial(upstream_media_runtime)]
fn bundled_media_clamps_output_compensation_to_freshness_budget() { fn bundled_media_preserves_large_measured_output_compensation_for_sync() {
temp_env::with_var( temp_env::with_var(
"LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", "LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS",
Some("20"), Some("20"),
@ -244,18 +244,64 @@ fn bundled_media_clamps_output_compensation_to_freshness_budget() {
assert_eq!( assert_eq!(
video.due_at.saturating_duration_since(audio.due_at), video.due_at.saturating_duration_since(audio.due_at),
Duration::from_millis(980), Duration::from_millis(1090),
"factory output compensation should be trimmed only enough to respect freshness" "measured output compensation is sync-critical and must not be clipped"
); );
assert!( assert!(
video.due_at.saturating_duration_since(now) <= Duration::from_secs(1), video.due_at.saturating_duration_since(now) > Duration::from_secs(1),
"clamped output compensation should preserve the live ceiling" "sync wins over freshness when measured UVC/UAC egress latency requires it"
); );
}); });
}, },
); );
} }
#[test]
#[serial(upstream_media_runtime)]
fn bundled_media_keeps_default_smooth_buffer_and_measured_output_compensation() {
temp_env::with_var_unset("LESAVKA_UPSTREAM_BUNDLED_PLAYOUT_DELAY_MS", || {
temp_env::with_var_unset("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", || {
temp_env::with_var("LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS", Some("1000"), || {
let runtime = UpstreamMediaRuntime::new();
runtime.set_playout_offsets(1_090_000, 0);
let _camera = runtime.activate_camera();
let _microphone = runtime.activate_microphone();
let now = tokio::time::Instant::now();
let epoch = now + Duration::from_millis(350);
let audio = play(runtime.plan_bundled_pts(
super::UpstreamMediaKind::Microphone,
1_000_000,
1,
1_000_000,
epoch,
));
let video = play(runtime.plan_bundled_pts(
super::UpstreamMediaKind::Camera,
1_000_000,
16_666,
1_000_000,
epoch,
));
assert!(
audio.due_at.saturating_duration_since(now) >= Duration::from_millis(300),
"the default bundled jitter buffer must remain available for smoothness"
);
assert_eq!(
video.due_at.saturating_duration_since(audio.due_at),
Duration::from_millis(1090),
"measured UVC/UAC egress compensation remains authoritative after the buffer"
);
assert!(
video.due_at.saturating_duration_since(now) > Duration::from_secs(1),
"sync-critical output delay may exceed the live budget because sync wins"
);
});
});
});
}
#[test] #[test]
#[serial(upstream_media_runtime)] #[serial(upstream_media_runtime)]
fn bundled_media_reanchors_future_wait_inside_one_second_freshness_budget() { fn bundled_media_reanchors_future_wait_inside_one_second_freshness_budget() {