bstein/lesavka
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

media: bias mjpeg uvc video for sync

Browse Source
This commit is contained in:
Brad Stein 2026-05-02 10:51:49 -03:00
parent 609517de03
commit 314c55b199
11 changed files with 69 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
AGENTS.md
View File

@ -231,3 +231,17 @@ Context: 0.16.x proved that queue tweaks and static calibration cannot guarantee
- 2026-05-02: 0.17.5 mirrored run still failed with insufficient paired evidence, and the client log still showed recurring `180-240ms` microphone packet age while camera age stayed near zero. Patch 0.17.6 splits oversized mic samples into `20ms` timestamped packets and keeps a short fresh server-side audio window instead of collapsing every pending burst to one newest chunk, aiming to preserve lip sync without making background audio choppy.
- 2026-05-02: 0.17.6 Bumblebee mirrored run proved Bumblebee mic packets are already `10ms`, but camera source timestamps were being rebased up to roughly `1.8s` into the future while mic packets sat around `180-240ms` old. Patch 0.17.7 adds a source lead cap (`80ms` default) to both direct and duration-paced client timestamp rebasing so bursty camera buffers cannot make the server wait for fake future video while fresh audio keeps moving.
- 2026-05-02: The launcher UI was still writing live control files with only camera/mic/speaker booleans, so media device combo changes were honestly only staged for the next child launch. Patch 0.17.7 extends the live media control file with base64-encoded camera source, camera profile, microphone source, and speaker sink choices; the relay child now rebuilds the affected camera, mic, or speaker pipeline when those selections change.
## 0.17.8 Sync-Only Output Bias Checklist
Context: 0.17.7 with the Bumblebee mic and BRIO camera removed the seconds-scale failure and left a stable browser-visible output skew: paired pulses were audio-late by roughly `+95ms` to `+183ms` (`median=+110.8ms`, `mean=+132.6ms`, `p95=+183.1ms`). Per user direction, 0.17.8 is only about establishing sync. Freshness and smoothness tuning are explicitly deferred until the mirrored probe is inside the sync band.
- [x] Do not change freshness ceilings, reanchor thresholds, queue policy, UAC smoothness, or startup healing behavior in this version.
- [x] Set the MJPEG/UVC factory video playout baseline to `+130ms` to counter the measured browser output audio-late bias.
- [x] Migrate only untouched old `0ms` and `+350ms` video defaults to the new `+130ms` baseline.
- [x] Preserve manual/site calibration values exactly as-is.
- [x] Update installer defaults so Theia receives the same `+130ms` baseline after reinstall.
- [x] Update docs and contracts to state the measured sync baseline clearly.
- [x] Run focused calibration/installer/runtime tests.
- [x] Run package checks before push.
- [x] Push clean semver `0.17.8` for installed client/server testing.

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lesavka_client"
version = "0.17.7"
version = "0.17.8"
dependencies = [
"anyhow",
"async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]]
name = "lesavka_common"
version = "0.17.7"
version = "0.17.8"
dependencies = [
"anyhow",
"base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]]
name = "lesavka_server"
version = "0.17.7"
version = "0.17.8"
dependencies = [
"anyhow",
"base64",

2
client/Cargo.toml
View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package]
name = "lesavka_client"
version = "0.17.7"
version = "0.17.8"
edition = "2024"
[dependencies]

2
common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "lesavka_common"
version = "0.17.7"
version = "0.17.8"
edition = "2024"
build = "build.rs"

2
docs/operational-env.md
View File

@ -254,7 +254,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta
| `LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS` | server upstream startup guard; paired startup must converge before this timeout or fail visibly, defaults to `60000` |
| `LESAVKA_UPSTREAM_STALE_DROP_MS` | server upstream freshness override; late audio/video that miss this budget are dropped instead of silently extending lag, defaults to `80` |
| `LESAVKA_UPSTREAM_TIMING_TRACE` | upstream capture/rebase trace override for sync debugging |
| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `350000` for MJPEG/UVC browser egress compensation |
| `LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US` | server upstream playout override; shifts webcam-video presentation relative to the shared playout epoch, defaults to `130000` for measured MJPEG/UVC browser sync compensation |
| `LESAVKA_UPLINK_CAMERA_PREVIEW` | client media capture/playback override |
| `LESAVKA_UPLINK_MIC_LEVEL` | client media capture/playback override |
| `LESAVKA_INSTALL_UVC_CODEC` | installer override; sets the persisted default UVC webcam codec in `/etc/lesavka/server.env` and `/etc/lesavka/uvc.env` |

7
scripts/install/server.sh
View File

@ -15,10 +15,11 @@ INSTALL_SERVER_BIND_ADDR=${LESAVKA_INSTALL_SERVER_BIND_ADDR:-0.0.0.0:50051}
LESAVKA_TLS_DIR=${LESAVKA_TLS_DIR:-/etc/lesavka/pki}
LESAVKA_CLIENT_BUNDLE=${LESAVKA_CLIENT_BUNDLE:-/etc/lesavka/lesavka-client-pki.tar.gz}
DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0
DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0
DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000
LEGACY_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=-45000
PREVIOUS_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=720000
PREVIOUS_TUNED_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=1260000
PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0
PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000
resolve_upstream_audio_playout_offset_us() {
@ -48,8 +49,8 @@ resolve_upstream_video_playout_offset_us() {
return 0
fi
if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then
echo "⚠️ migrating stale upstream video playout offset to the 0.17 freshness-first planner default." >&2
if [[ ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" || ${LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US:-} == "$PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US" ]]; then
echo "⚠️ migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline." >&2
echo " Use LESAVKA_INSTALL_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US to intentionally keep an older value." >&2
printf '%s\n' "$DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US"
return 0

2
server/Cargo.toml
View File

@ -10,7 +10,7 @@ bench = false
[package]
name = "lesavka_server"
version = "0.17.7"
version = "0.17.8"
edition = "2024"
autobins = false

55
server/src/calibration.rs
View File

@ -10,7 +10,10 @@ use lesavka_common::lesavka::{
use crate::upstream_media_runtime::UpstreamMediaRuntime;
pub const FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 0;
pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 0;
// 0.17.7's mirrored browser probe measured a stable MJPEG/UVC output-path
// bias where browser audio arrived about 130ms after video. Delay video by
// that small factory baseline; freshness policy stays owned by the planner.
pub const FACTORY_MJPEG_VIDEO_OFFSET_US: i64 = 130_000;
const LEGACY_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = -45_000;
const PREVIOUS_FACTORY_MJPEG_AUDIO_OFFSET_US: i64 = 720_000;
const PREVIOUS_TUNED_MJPEG_AUDIO_OFFSET_US: i64 = 1_260_000;
@ -340,7 +343,7 @@ mod tests {
|| {
let state = snapshot_from_env();
assert_eq!(state.default_audio_offset_us, 0);
assert_eq!(state.active_video_offset_us, 0);
assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.source, "factory");
},
);
@ -364,7 +367,10 @@ mod tests {
})
.expect("manual adjust applies");
assert_eq!(state.active_audio_offset_us, -5_000);
assert_eq!(runtime.playout_offsets(), (0, -5_000));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US, -5_000)
);
let raw = std::fs::read_to_string(file.path()).expect("persisted calibration");
assert!(raw.contains("active_audio_offset_us=-5000"));
});
@ -449,10 +455,13 @@ mod tests {
let state = store.current();
assert_eq!(state.active_audio_offset_us, 0);
assert_eq!(state.default_audio_offset_us, 0);
assert_eq!(state.active_video_offset_us, 0);
assert_eq!(state.default_video_offset_us, 0);
assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.source, "factory");
assert_eq!(runtime.playout_offsets(), (0, 0));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US, 0)
);
assert!(state.detail.contains("migrated legacy MJPEG"));
});
}
@ -481,11 +490,14 @@ mod tests {
let state = store.current();
assert_eq!(state.active_audio_offset_us, 0);
assert_eq!(state.default_audio_offset_us, 0);
assert_eq!(state.active_video_offset_us, 0);
assert_eq!(state.default_video_offset_us, 0);
assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.source, "factory");
assert_eq!(runtime.playout_offsets(), (0, 0));
assert!(state.detail.contains("to audio +0.0ms/video +0.0ms"));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US, 0)
);
assert!(state.detail.contains("to audio +0.0ms/video +130.0ms"));
});
}
@ -513,10 +525,13 @@ mod tests {
let state = store.current();
assert_eq!(state.active_audio_offset_us, 0);
assert_eq!(state.default_audio_offset_us, 0);
assert_eq!(state.active_video_offset_us, 0);
assert_eq!(state.default_video_offset_us, 0);
assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.source, "factory");
assert_eq!(runtime.playout_offsets(), (0, 0));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US, 0)
);
});
}
@ -544,10 +559,13 @@ mod tests {
let state = store.current();
assert_eq!(state.active_audio_offset_us, 0);
assert_eq!(state.default_audio_offset_us, 0);
assert_eq!(state.active_video_offset_us, 0);
assert_eq!(state.default_video_offset_us, 0);
assert_eq!(state.active_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.default_video_offset_us, FACTORY_MJPEG_VIDEO_OFFSET_US);
assert_eq!(state.source, "factory");
assert_eq!(runtime.playout_offsets(), (0, 0));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US, 0)
);
});
}
@ -604,7 +622,10 @@ mod tests {
.expect("blind estimate");
assert_eq!(blind.source, "blind");
assert!(blind.detail.contains("delivery skew 44.0ms"));
assert_eq!(runtime.playout_offsets(), (-2_000, 5_000));
assert_eq!(
runtime.playout_offsets(),
(FACTORY_MJPEG_VIDEO_OFFSET_US - 2_000, 5_000)
);
let manual = store
.apply(CalibrationRequest {

3
server/src/upstream_media_runtime/tests/config.rs
View File

@ -1,4 +1,5 @@
use super::UpstreamMediaKind;
use crate::calibration::FACTORY_MJPEG_VIDEO_OFFSET_US;
use serial_test::serial;
use std::time::Duration;
@ -74,7 +75,7 @@ fn upstream_playout_offsets_default_to_mjpeg_calibration_and_accept_overrides()
);
assert_eq!(
super::upstream_playout_offset_us(UpstreamMediaKind::Camera),
0
FACTORY_MJPEG_VIDEO_OFFSET_US
);
});
});

5
testing/tests/server_install_script_contract.rs
View File

@ -55,7 +55,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_MAX_LIVE_LAG_MS:-1000}"));
assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_STARTUP_TIMEOUT_MS:-60000}"));
assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=0"));
assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0"));
assert!(SERVER_INSTALL.contains("DEFAULT_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=130000"));
assert!(
SERVER_INSTALL.contains("resolve_upstream_video_playout_offset_us"),
"video offset should be resolved through stale-baseline migration logic"
@ -64,6 +64,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
assert!(
SERVER_INSTALL.contains("PREVIOUS_TUNED_MJPEG_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US=1260000")
);
assert!(SERVER_INSTALL.contains("PREVIOUS_ZERO_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=0"));
assert!(
SERVER_INSTALL.contains("PREVIOUS_DELAYED_MJPEG_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US=350000")
);
@ -80,7 +81,7 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
"installer should not preserve old MJPEG/UVC sync baselines accidentally"
);
assert!(
SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 freshness-first planner default"),
SERVER_INSTALL.contains("migrating stale upstream video playout offset to the 0.17 measured MJPEG/UVC sync baseline"),
"installer should not preserve old video delay baselines accidentally"
);
assert!(SERVER_INSTALL.contains("${LESAVKA_UPSTREAM_PAIR_SLACK_US:-80000}"));

2
testing/tests/server_main_rpc_contract.rs
View File

@ -487,7 +487,7 @@ mod server_main_rpc {
.into_inner();
assert_eq!(adjusted.source, "blind");
assert_eq!(adjusted.active_audio_offset_us, 10_000);
assert_eq!(adjusted.active_video_offset_us, 2_000);
assert_eq!(adjusted.active_video_offset_us, 132_000);
assert!(
std::fs::read_to_string(calibration_path)
.expect("persisted")