From 0ec6e0c7014c76ffaa2b1e2fa3ca1f47c31eee91 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Sat, 2 May 2026 12:49:38 -0300 Subject: [PATCH] media: compensate browser-visible av sync delay --- AGENTS.md | 15 ++++ Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- docs/operational-env.md | 2 +- scripts/install/server.sh | 7 +- server/Cargo.toml | 2 +- server/src/calibration.rs | 72 +++++++++++++++---- server/src/upstream_media_runtime.rs | 16 ++++- .../upstream_media_runtime/tests/planning.rs | 28 ++++++++ .../tests/server_install_script_contract.rs | 7 +- 11 files changed, 130 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 40269b4..864ddfe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -271,3 +271,18 @@ Context: 0.17.9 installed cleanly on both ends (`fbf274d`) and improved the mirr - [x] Run focused upstream planner tests. - [x] Run package checks before push. - [x] Push clean semver `0.17.10` for installed client/server testing. + +## 0.17.11 Sync-Only Browser Egress Compensation Checklist + +Context: 0.17.10 installed cleanly on both ends (`4bb0f4a`) and produced a high-confidence coded-pulse failure instead of probe ambiguity. Browser-visible audio on Tethys arrived about `+891ms` to `+971ms` after the matching video (`median=+962.1ms`, `mean=+946.7ms`, `p95=+971.5ms`), while the server planner reported internal skew near zero (`planner_skew_ms=-56.9`). The missing model is UAC/browser output egress latency: Lesavka was treating `appsrc.push_buffer`/UAC enqueue as audio presentation, but the browser consumes that audio about one second later. + +- [x] Keep 0.17.11 scoped to establishing sync; do not tune freshness ceilings or smoothness policy. +- [x] Raise the MJPEG/UVC factory video playout baseline from `+130ms` to `+1090ms` to align video with browser-visible UAC audio. +- [x] Allow intentional A/V playout offsets to exceed the generic future-wait freshness guard so the planner does not immediately reanchor away the sync compensation. +- [x] Widen calibration offset bounds so the measured browser egress baseline is representable instead of silently clamped. +- [x] Migrate untouched `0ms`, `+130ms`, and `+350ms` MJPEG/UVC video baselines to the new browser-visible baseline. +- [x] Preserve manual/site calibration values exactly as-is. +- [x] Update installer defaults so Theia receives the same browser-visible baseline after reinstall. +- [x] Update operational docs and installer contract tests for the new baseline. +- [x] Run focused calibration, installer, and runtime checks. +- [ ] Push clean semver `0.17.11` for installed client/server testing. diff --git a/Cargo.lock b/Cargo.lock index a9bb705..d8bb354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.17.10" +version = "0.17.11" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.17.10" +version = "0.17.11" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.17.10" +version = "0.17.11" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 266d309..f99c99a 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.17.10" +version = "0.17.11" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 3e3f86c..b252dfe 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.17.10" +version = "0.17.11" edition = "2024" build = "build.rs" diff --git a/docs/operational-env.md b/docs/operational-env.md index 9341eef..3f09295 100644 --- a/docs/operational-env.md +++ b/docs/operational-env.md @@ -255,7 +255,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS` | server upstream startup guard; paired startup must converge before this timeout or fail visibly, defaults to `60000` | | `LESAVKA_UPSTREAM_STALE_DROP_MS` | server upstream freshness override; late audio/video that miss this budget are dropped instead of silently extending lag, defaults to `80` | | `LESAVKA_UPSTREAM_TIMING_TRACE` | upstream capture/rebase trace override for sync debugging | -| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `130000` for measured MJPEG/UVC browser sync compensation | +| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `1090000` for measured MJPEG/UVC browser-visible sync compensation | | `LESAVKA_UPLINK_CAMERA_PREVIEW` | client media capture/playback override | | `LESAVKA_UPLINK_MIC_LEVEL` | client media capture/playback override | | `LESAVKA_INSTALL_UVC_CODEC` | installer override; sets the persisted default UVC webcam codec in `/etc/lesavka/server.env` and `/etc/lesavka/uvc.env` | diff --git a/scripts/install/server.sh b/scripts/install/server.sh index 88c0e43..c2fb90e 100755 --- a/scripts/install/server.sh +++ b/scripts/install/server.sh @@ -15,12 +15,13 @@ INSTALL_SERVER_BIND_ADDR=${LESAVKA_INSTALL_SERVER_BIND_ADDR:-0.0.0.0:50051} LESAVKA_TLS_DIR=${LESAVKA_TLS_DIR:-/etc/lesavka/pki} LESAVKA_CLIENT_BUNDLE=${LESAVKA_CLIENT_BUNDLE:-/etc/lesavka/lesavka-client-pki.tar.gz} DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0 -DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000 +DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=1090000 LEGACY_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=-45000 PREVIOUS_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=720000 PREVIOUS_TUNED_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=1260000 PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0 PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000 +PREVIOUS_BROWSER_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000 resolve_upstream_audio_playout_offset_us() { if [[ -n ${LESAVKA_INSTALL_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US:-} ]]; then @@ -49,8 +50,8 @@ resolve_upstream_video_playout_offset_us() { return 0 fi - if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" || ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then - echo "⚠️ migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline." >&2 + if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" || ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" || ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_BROWSER_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then + echo "⚠️ migrating stale upstream video playout offset to the 0.17 browser-visible MJPEG/UVC sync baseline." >&2 echo " Use LESAVKA_INSTALL_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US to intentionally keep an older value." >&2 printf '%s\n' "$DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" return 0 diff --git a/server/Cargo.toml b/server/Cargo.toml index 2694cd5..a33e620 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.17.10" +version = "0.17.11" edition = "2024" autobins = false diff --git a/server/src/calibration.rs b/server/src/calibration.rs index e3fd3fc..13c2cfd 100644 --- a/server/src/calibration.rs +++ b/server/src/calibration.rs @@ -10,18 +10,22 @@ use lesavka_common::lesavka::{ use crate::upstream_media_runtime::UpstreamMediaRuntime; pub const FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 0; -// 0.17.7's mirrored browser probe measured a stable MJPEG/UVC output-path -// bias where browser audio arrived about 130ms after video. Delay video by -// that small factory baseline; freshness policy stays owned by the planner. -pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 130_000; +// 0.17.10's mirrored browser probe showed the remaining sync error lives +// after server enqueue: UAC audio becomes browser-visible on Tethys roughly +// 950-970ms after the matching UVC frame. Delay MJPEG/UVC video by that +// measured egress delta; the planner treats this as intentional output-path +// sync compensation rather than stale-media drift. +pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 1_090_000; const LEGACY_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = -45_000; const PREVIOUS_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 720_000; const PREVIOUS_TUNED_MJPEG_AUDIO_OFFSET_US: i64 = 1_260_000; const PREVIOUS_FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 0; const PREVIOUS_DELAYED_FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 350_000; +const PREVIOUS_BROWSER_FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 130_000; const PROFILE: &str = "mjpeg"; const FACTORY_CONFIDENCE: &str = "factory"; -const OFFSET_LIMIT_US: i64 = 500_000; +const PREVIOUS_OFFSET_LIMIT_US: i64 = 500_000; +const OFFSET_LIMIT_US: i64 = 1_500_000; #[derive(Debug, Clone, PartialEq, Eq)] struct CalibrationSnapshot { @@ -243,10 +247,12 @@ fn parse_snapshot(raw: &str) -> CalibrationSnapshot { fn migrate_legacy_snapshot(mut state: CalibrationSnapshot) -> CalibrationSnapshot { let source_allows_migration = matches!(state.source.as_str(), "factory" | "env"); let confidence_allows_migration = matches!(state.confidence.as_str(), "factory" | "configured"); - let clamped_previous_baseline = state.default_audio_offset_us == OFFSET_LIMIT_US - && state - .detail - .contains("loaded upstream A/V calibration defaults"); + let clamped_previous_baseline = matches!( + state.default_audio_offset_us, + PREVIOUS_OFFSET_LIMIT_US | OFFSET_LIMIT_US + ) && state + .detail + .contains("loaded upstream A/V calibration defaults"); let untouched_legacy_audio = (matches!( state.default_audio_offset_us, FACTORY_MJPEG_AUDIO_OFFSET_US @@ -257,7 +263,9 @@ fn migrate_legacy_snapshot(mut state: CalibrationSnapshot) -> CalibrationSnapsho && state.active_audio_offset_us == state.default_audio_offset_us; let untouched_legacy_video = matches!( state.default_video_offset_us, - PREVIOUS_FACTORY_MJPEG_VIDEO_OFFSET_US | PREVIOUS_DELAYED_FACTORY_MJPEG_VIDEO_OFFSET_US + PREVIOUS_FACTORY_MJPEG_VIDEO_OFFSET_US + | PREVIOUS_DELAYED_FACTORY_MJPEG_VIDEO_OFFSET_US + | PREVIOUS_BROWSER_FACTORY_MJPEG_VIDEO_OFFSET_US ) && state.active_video_offset_us == state.default_video_offset_us; if state.profile == PROFILE && source_allows_migration @@ -395,7 +403,7 @@ mod tests { ], || { let state = snapshot_from_env(); - assert_eq!(state.default_audio_offset_us, -500_000); + assert_eq!(state.default_audio_offset_us, -OFFSET_LIMIT_US); assert_eq!(state.default_video_offset_us, 12_345); assert_eq!(state.source, "env"); assert_eq!(state.confidence, "configured"); @@ -423,7 +431,7 @@ mod tests { ); assert_eq!(state.default_audio_offset_us, FACTORY_MJPEG_AUDIO_OFFSET_US); assert_eq!(state.default_video_offset_us, 2_500); - assert_eq!(state.active_audio_offset_us, -500_000); + assert_eq!(state.active_audio_offset_us, -OFFSET_LIMIT_US); assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "saved"); assert_eq!(state.confidence, FACTORY_CONFIDENCE); @@ -497,7 +505,41 @@ mod tests { runtime.playout_offsets(), (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) ); - assert!(state.detail.contains("to audio +0.0ms/video +130.0ms")); + assert!(state.detail.contains("to audio +0.0ms/video +1090.0ms")); + }); + } + + #[test] + fn load_migrates_browser_video_factory_mjpeg_baseline() { + let file = NamedTempFile::new().expect("temp calibration file"); + std::fs::write( + file.path(), + r#" + profile="mjpeg" + default_audio_offset_us=0 + default_video_offset_us=130000 + active_audio_offset_us=0 + active_video_offset_us=130000 + source="factory" + confidence="factory" + detail="loaded upstream A/V calibration defaults" + "#, + ) + .expect("browser video calibration seed"); + let path = file.path().to_string_lossy().to_string(); + temp_env::with_var("LESAVKA_CALIBRATION_PATH", Some(path.as_str()), || { + let runtime = Arc::new(UpstreamMediaRuntime::new()); + let store = CalibrationStore::load(runtime.clone()); + let state = store.current(); + assert_eq!(state.active_audio_offset_us, 0); + assert_eq!(state.default_audio_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.source, "factory"); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) + ); }); } @@ -630,14 +672,14 @@ mod tests { let manual = store .apply(CalibrationRequest { action: CalibrationAction::AdjustActive as i32, - audio_delta_us: 999_999, + audio_delta_us: 1_999_999, video_delta_us: 0, observed_delivery_skew_ms: 0.0, observed_enqueue_skew_ms: 0.0, note: String::new(), }) .expect("manual clamp"); - assert_eq!(manual.active_audio_offset_us, 500_000); + assert_eq!(manual.active_audio_offset_us, OFFSET_LIMIT_US); let saved = store .apply(CalibrationRequest { diff --git a/server/src/upstream_media_runtime.rs b/server/src/upstream_media_runtime.rs index 4e433eb..836d1e7 100644 --- a/server/src/upstream_media_runtime.rs +++ b/server/src/upstream_media_runtime.rs @@ -102,6 +102,13 @@ impl UpstreamMediaRuntime { camera_offset_us.saturating_sub(microphone_offset_us).max(0) as u64 } + fn intentional_future_wait_allowance_us(&self, kind: UpstreamMediaKind) -> u64 { + match kind { + UpstreamMediaKind::Camera => self.audio_ahead_video_allowance_us(), + UpstreamMediaKind::Microphone => self.positive_audio_delay_allowance_us(), + } + } + /// Mark one audio chunk as actually handed to the UAC sink. pub fn mark_audio_presented(&self, local_pts_us: u64) { let mut state = self @@ -489,7 +496,11 @@ impl UpstreamMediaRuntime { apply_playout_offset(epoch + Duration::from_micros(local_pts_us), sink_offset_us); let mut late_by = now.checked_duration_since(due_at).unwrap_or_default(); let reanchor_threshold = upstream_reanchor_late_threshold(playout_delay); - let max_future_wait = max_live_lag.saturating_sub(source_lag); + let intentional_future_wait_allowance = + Duration::from_micros(self.intentional_future_wait_allowance_us(kind)); + let max_future_wait = max_live_lag + .saturating_sub(source_lag) + .saturating_add(intentional_future_wait_allowance); let due_future_wait = due_at.saturating_duration_since(now); if late_by > reanchor_threshold || due_future_wait > max_future_wait { let old_late_by = late_by; @@ -527,7 +538,8 @@ impl UpstreamMediaRuntime { } let predicted_lag_at_playout = source_lag.saturating_add(due_at.saturating_duration_since(now)); - if predicted_lag_at_playout > max_live_lag { + if predicted_lag_at_playout > max_live_lag.saturating_add(intentional_future_wait_allowance) + { match kind { UpstreamMediaKind::Camera => { state.stale_video_drops = state.stale_video_drops.saturating_add(1); diff --git a/server/src/upstream_media_runtime/tests/planning.rs b/server/src/upstream_media_runtime/tests/planning.rs index 0c79ed0..19c0f47 100644 --- a/server/src/upstream_media_runtime/tests/planning.rs +++ b/server/src/upstream_media_runtime/tests/planning.rs @@ -397,6 +397,34 @@ fn configured_video_delay_does_not_make_the_planner_freeze_video() { }); } +#[test] +#[serial(upstream_media_runtime)] +fn browser_visible_video_delay_is_not_reanchored_away() { + temp_env::with_var("LESAVKA_UPSTREAM_PLAYOUT_DELAY_MS", Some("350"), || { + temp_env::with_var("LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS", Some("1000"), || { + let runtime = runtime_without_offsets(); + runtime.set_playout_offsets(1_090_000, 0); + let _camera = runtime.activate_camera(); + let _microphone = runtime.activate_microphone(); + + assert!(matches!( + runtime.plan_video_pts(1_000_000, 16_666), + super::UpstreamPlanDecision::AwaitingPair + )); + let audio = play(runtime.plan_audio_pts(1_000_000)); + let video = play(runtime.plan_video_pts(1_000_000, 16_666)); + + assert!( + video.due_at.saturating_duration_since(audio.due_at) >= Duration::from_millis(1080), + "intentional browser-visible video delay must survive the freshness reanchor" + ); + let snapshot = runtime.snapshot(); + assert_eq!(snapshot.freshness_reanchors, 0); + assert_eq!(snapshot.stale_video_drops, 0); + }); + }); +} + #[test] #[serial(upstream_media_runtime)] fn paired_startup_times_out_instead_of_waiting_forever() { diff --git a/testing/tests/server_install_script_contract.rs b/testing/tests/server_install_script_contract.rs index efc5c11..50e8d79 100644 --- a/testing/tests/server_install_script_contract.rs +++ b/testing/tests/server_install_script_contract.rs @@ -56,7 +56,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS:-1000}")); assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS:-60000}")); assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0")); - assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000")); + assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=1090000")); assert!( SERVER_INSTALL.contains("resolve_upstream_video_playout_offset_us"), "video offset should be resolved through stale-baseline migration logic" @@ -69,6 +69,9 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { assert!( SERVER_INSTALL.contains("PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000") ); + assert!( + SERVER_INSTALL.contains("PREVIOUS_BROWSER_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000") + ); assert!( SERVER_INSTALL.contains("LESAVKA_INSTALL_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US"), "install-specific offset override should bypass stale ambient runtime env" @@ -82,7 +85,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { "installer should not preserve old MJPEG/UVC sync baselines accidentally" ); assert!( - SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline"), + SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 browser-visible MJPEG/UVC sync baseline"), "installer should not preserve old video delay baselines accidentally" ); assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_PAIR_SLACK_US:-80000}"));