From 314c55b199535800aec5a9178412aba4791791f0 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Sat, 2 May 2026 10:51:49 -0300 Subject: [PATCH] media: bias mjpeg uvc video for sync --- AGENTS.md | 14 +++++ Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- docs/operational-env.md | 2 +- scripts/install/server.sh | 7 ++- server/Cargo.toml | 2 +- server/src/calibration.rs | 55 +++++++++++++------ .../upstream_media_runtime/tests/config.rs | 3 +- .../tests/server_install_script_contract.rs | 5 +- testing/tests/server_main_rpc_contract.rs | 2 +- 11 files changed, 69 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 956ee71..21189e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -231,3 +231,17 @@ Context: 0.16.x proved that queue tweaks and static calibration cannot guarantee - 2026-05-02: 0.17.5 mirrored run still failed with insufficient paired evidence, and the client log still showed recurring `180-240ms` microphone packet age while camera age stayed near zero. Patch 0.17.6 splits oversized mic samples into `20ms` timestamped packets and keeps a short fresh server-side audio window instead of collapsing every pending burst to one newest chunk, aiming to preserve lip sync without making background audio choppy. - 2026-05-02: 0.17.6 Bumblebee mirrored run proved Bumblebee mic packets are already `10ms`, but camera source timestamps were being rebased up to roughly `1.8s` into the future while mic packets sat around `180-240ms` old. Patch 0.17.7 adds a source lead cap (`80ms` default) to both direct and duration-paced client timestamp rebasing so bursty camera buffers cannot make the server wait for fake future video while fresh audio keeps moving. - 2026-05-02: The launcher UI was still writing live control files with only camera/mic/speaker booleans, so media device combo changes were honestly only staged for the next child launch. Patch 0.17.7 extends the live media control file with base64-encoded camera source, camera profile, microphone source, and speaker sink choices; the relay child now rebuilds the affected camera, mic, or speaker pipeline when those selections change. + +## 0.17.8 Sync-Only Output Bias Checklist + +Context: 0.17.7 with the Bumblebee mic and BRIO camera removed the seconds-scale failure and left a stable browser-visible output skew: paired pulses were audio-late by roughly `+95ms` to `+183ms` (`median=+110.8ms`, `mean=+132.6ms`, `p95=+183.1ms`). Per user direction, 0.17.8 is only about establishing sync. Freshness and smoothness tuning are explicitly deferred until the mirrored probe is inside the sync band. + +- [x] Do not change freshness ceilings, reanchor thresholds, queue policy, UAC smoothness, or startup healing behavior in this version. +- [x] Set the MJPEG/UVC factory video playout baseline to `+130ms` to counter the measured browser output audio-late bias. +- [x] Migrate only untouched old `0ms` and `+350ms` video defaults to the new `+130ms` baseline. +- [x] Preserve manual/site calibration values exactly as-is. +- [x] Update installer defaults so Theia receives the same `+130ms` baseline after reinstall. +- [x] Update docs and contracts to state the measured sync baseline clearly. +- [x] Run focused calibration/installer/runtime tests. +- [x] Run package checks before push. +- [x] Push clean semver `0.17.8` for installed client/server testing. diff --git a/Cargo.lock b/Cargo.lock index 2c1b573..e61e313 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.17.7" +version = "0.17.8" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.17.7" +version = "0.17.8" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.17.7" +version = "0.17.8" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index c3b6c69..07ac200 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.17.7" +version = "0.17.8" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 424ded3..e82b14f 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.17.7" +version = "0.17.8" edition = "2024" build = "build.rs" diff --git a/docs/operational-env.md b/docs/operational-env.md index dc58e44..0b3cc0c 100644 --- a/docs/operational-env.md +++ b/docs/operational-env.md @@ -254,7 +254,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta | `LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS` | server upstream startup guard; paired startup must converge before this timeout or fail visibly, defaults to `60000` | | `LESAVKA_UPSTREAM_STALE_DROP_MS` | server upstream freshness override; late audio/video that miss this budget are dropped instead of silently extending lag, defaults to `80` | | `LESAVKA_UPSTREAM_TIMING_TRACE` | upstream capture/rebase trace override for sync debugging | -| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `350000` for MJPEG/UVC browser egress compensation | +| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `130000` for measured MJPEG/UVC browser sync compensation | | `LESAVKA_UPLINK_CAMERA_PREVIEW` | client media capture/playback override | | `LESAVKA_UPLINK_MIC_LEVEL` | client media capture/playback override | | `LESAVKA_INSTALL_UVC_CODEC` | installer override; sets the persisted default UVC webcam codec in `/etc/lesavka/server.env` and `/etc/lesavka/uvc.env` | diff --git a/scripts/install/server.sh b/scripts/install/server.sh index 32b24e0..632d54b 100755 --- a/scripts/install/server.sh +++ b/scripts/install/server.sh @@ -15,10 +15,11 @@ INSTALL_SERVER_BIND_ADDR=${LESAVKA_INSTALL_SERVER_BIND_ADDR:-0.0.0.0:50051} LESAVKA_TLS_DIR=${LESAVKA_TLS_DIR:-/etc/lesavka/pki} LESAVKA_CLIENT_BUNDLE=${LESAVKA_CLIENT_BUNDLE:-/etc/lesavka/lesavka-client-pki.tar.gz} DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0 -DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0 +DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000 LEGACY_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=-45000 PREVIOUS_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=720000 PREVIOUS_TUNED_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=1260000 +PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0 PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000 resolve_upstream_audio_playout_offset_us() { @@ -48,8 +49,8 @@ resolve_upstream_video_playout_offset_us() { return 0 fi - if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then - echo "⚠️ migrating stale upstream video playout offset to the 0.17 freshness-first planner default." >&2 + if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" || ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then + echo "⚠️ migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline." >&2 echo " Use LESAVKA_INSTALL_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US to intentionally keep an older value." >&2 printf '%s\n' "$DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" return 0 diff --git a/server/Cargo.toml b/server/Cargo.toml index 0b340b9..cbb438e 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.17.7" +version = "0.17.8" edition = "2024" autobins = false diff --git a/server/src/calibration.rs b/server/src/calibration.rs index d25531f..e3fd3fc 100644 --- a/server/src/calibration.rs +++ b/server/src/calibration.rs @@ -10,7 +10,10 @@ use lesavka_common::lesavka::{ use crate::upstream_media_runtime::UpstreamMediaRuntime; pub const FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 0; -pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 0; +// 0.17.7's mirrored browser probe measured a stable MJPEG/UVC output-path +// bias where browser audio arrived about 130ms after video. Delay video by +// that small factory baseline; freshness policy stays owned by the planner. +pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 130_000; const LEGACY_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = -45_000; const PREVIOUS_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 720_000; const PREVIOUS_TUNED_MJPEG_AUDIO_OFFSET_US: i64 = 1_260_000; @@ -340,7 +343,7 @@ mod tests { || { let state = snapshot_from_env(); assert_eq!(state.default_audio_offset_us, 0); - assert_eq!(state.active_video_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "factory"); }, ); @@ -364,7 +367,10 @@ mod tests { }) .expect("manual adjust applies"); assert_eq!(state.active_audio_offset_us, -5_000); - assert_eq!(runtime.playout_offsets(), (0, -5_000)); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, -5_000) + ); let raw = std::fs::read_to_string(file.path()).expect("persisted calibration"); assert!(raw.contains("active_audio_offset_us=-5000")); }); @@ -449,10 +455,13 @@ mod tests { let state = store.current(); assert_eq!(state.active_audio_offset_us, 0); assert_eq!(state.default_audio_offset_us, 0); - assert_eq!(state.active_video_offset_us, 0); - assert_eq!(state.default_video_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "factory"); - assert_eq!(runtime.playout_offsets(), (0, 0)); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) + ); assert!(state.detail.contains("migrated legacy MJPEG")); }); } @@ -481,11 +490,14 @@ mod tests { let state = store.current(); assert_eq!(state.active_audio_offset_us, 0); assert_eq!(state.default_audio_offset_us, 0); - assert_eq!(state.active_video_offset_us, 0); - assert_eq!(state.default_video_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "factory"); - assert_eq!(runtime.playout_offsets(), (0, 0)); - assert!(state.detail.contains("to audio +0.0ms/video +0.0ms")); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) + ); + assert!(state.detail.contains("to audio +0.0ms/video +130.0ms")); }); } @@ -513,10 +525,13 @@ mod tests { let state = store.current(); assert_eq!(state.active_audio_offset_us, 0); assert_eq!(state.default_audio_offset_us, 0); - assert_eq!(state.active_video_offset_us, 0); - assert_eq!(state.default_video_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "factory"); - assert_eq!(runtime.playout_offsets(), (0, 0)); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) + ); }); } @@ -544,10 +559,13 @@ mod tests { let state = store.current(); assert_eq!(state.active_audio_offset_us, 0); assert_eq!(state.default_audio_offset_us, 0); - assert_eq!(state.active_video_offset_us, 0); - assert_eq!(state.default_video_offset_us, 0); + assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); + assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US); assert_eq!(state.source, "factory"); - assert_eq!(runtime.playout_offsets(), (0, 0)); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US, 0) + ); }); } @@ -604,7 +622,10 @@ mod tests { .expect("blind estimate"); assert_eq!(blind.source, "blind"); assert!(blind.detail.contains("delivery skew 44.0ms")); - assert_eq!(runtime.playout_offsets(), (-2_000, 5_000)); + assert_eq!( + runtime.playout_offsets(), + (FACTORY_MJPEG_VIDEO_OFFSET_US - 2_000, 5_000) + ); let manual = store .apply(CalibrationRequest { diff --git a/server/src/upstream_media_runtime/tests/config.rs b/server/src/upstream_media_runtime/tests/config.rs index eb2388e..eeff580 100644 --- a/server/src/upstream_media_runtime/tests/config.rs +++ b/server/src/upstream_media_runtime/tests/config.rs @@ -1,4 +1,5 @@ use super::UpstreamMediaKind; +use crate::calibration::FACTORY_MJPEG_VIDEO_OFFSET_US; use serial_test::serial; use std::time::Duration; @@ -74,7 +75,7 @@ fn upstream_playout_offsets_default_to_mjpeg_calibration_and_accept_overrides() ); assert_eq!( super::upstream_playout_offset_us(UpstreamMediaKind::Camera), - 0 + FACTORY_MJPEG_VIDEO_OFFSET_US ); }); }); diff --git a/testing/tests/server_install_script_contract.rs b/testing/tests/server_install_script_contract.rs index 6bff9bd..1c994f4 100644 --- a/testing/tests/server_install_script_contract.rs +++ b/testing/tests/server_install_script_contract.rs @@ -55,7 +55,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS:-1000}")); assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS:-60000}")); assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0")); - assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0")); + assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000")); assert!( SERVER_INSTALL.contains("resolve_upstream_video_playout_offset_us"), "video offset should be resolved through stale-baseline migration logic" @@ -64,6 +64,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { assert!( SERVER_INSTALL.contains("PREVIOUS_TUNED_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=1260000") ); + assert!(SERVER_INSTALL.contains("PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0")); assert!( SERVER_INSTALL.contains("PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000") ); @@ -80,7 +81,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { "installer should not preserve old MJPEG/UVC sync baselines accidentally" ); assert!( - SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 freshness-first planner default"), + SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline"), "installer should not preserve old video delay baselines accidentally" ); assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_PAIR_SLACK_US:-80000}")); diff --git a/testing/tests/server_main_rpc_contract.rs b/testing/tests/server_main_rpc_contract.rs index fcdab23..cfa470f 100644 --- a/testing/tests/server_main_rpc_contract.rs +++ b/testing/tests/server_main_rpc_contract.rs @@ -487,7 +487,7 @@ mod server_main_rpc { .into_inner(); assert_eq!(adjusted.source, "blind"); assert_eq!(adjusted.active_audio_offset_us, 10_000); - assert_eq!(adjusted.active_video_offset_us, 2_000); + assert_eq!(adjusted.active_video_offset_us, 132_000); assert!( std::fs::read_to_string(calibration_path) .expect("persisted")