diff --git a/Cargo.lock b/Cargo.lock index b8dbc29..40235a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.16.11" +version = "0.16.12" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.16.11" +version = "0.16.12" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.16.11" +version = "0.16.12" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 3c8d722..60ab076 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.16.11" +version = "0.16.12" edition = "2024" [dependencies] diff --git a/client/src/bin/lesavka-sync-analyze.rs b/client/src/bin/lesavka-sync-analyze.rs index ff8e60c..fd5f002 100644 --- a/client/src/bin/lesavka-sync-analyze.rs +++ b/client/src/bin/lesavka-sync-analyze.rs @@ -123,6 +123,7 @@ A/V sync report for {capture} - video onsets: {video_events} - audio onsets: {audio_events} - paired pulses: {paired_events} +- activity start delta: {activity_start_delta:+.1} ms (audio after video is positive) - first skew: {first_skew:+.1} ms (audio after video is positive) - last skew: {last_skew:+.1} ms - mean skew: {mean_skew:+.1} ms @@ -142,6 +143,7 @@ A/V sync report for {capture} video_events = report.video_event_count, audio_events = report.audio_event_count, paired_events = report.paired_event_count, + activity_start_delta = report.activity_start_delta_ms, first_skew = report.first_skew_ms, last_skew = report.last_skew_ms, mean_skew = report.mean_skew_ms, diff --git a/client/src/sync_probe/analyze/onset_detection/correlation.rs b/client/src/sync_probe/analyze/onset_detection/correlation.rs index 087b4e8..b86290a 100644 --- a/client/src/sync_probe/analyze/onset_detection/correlation.rs +++ b/client/src/sync_probe/analyze/onset_detection/correlation.rs @@ -33,6 +33,7 @@ pub(super) fn correlate_onsets( bail!("pulse period must stay positive"); } + let activity_start_delta_ms = (audio_onsets_s[0] - video_onsets_s[0]) * 1000.0; let (video_onsets_s, audio_onsets_s, common_window) = trim_onsets_to_common_activity_window(video_onsets_s, audio_onsets_s, max_pair_gap_s); let expected_start_skew_ms = (audio_onsets_s[0] - video_onsets_s[0]) * 1000.0; @@ -69,6 +70,7 @@ pub(super) fn correlate_onsets( Ok(sync_report_from_pairs( common_window.filter_onsets(video_onsets_s), common_window.filter_onsets(audio_onsets_s), + activity_start_delta_ms, pairs, )) } @@ -121,6 +123,7 @@ pub(crate) fn correlate_segments( bail!("audio onset list is empty"); } + let activity_start_delta_ms = (audio_onsets_s[0] - video_onsets_s[0]) * 1000.0; let (video_onsets_s, audio_onsets_s, common_window) = trim_onsets_to_common_activity_window(&video_onsets_s, &audio_onsets_s, max_pair_gap_s); let expected_start_skew_ms = (audio_onsets_s[0] - video_onsets_s[0]) * 1000.0; @@ -166,6 +169,7 @@ pub(crate) fn correlate_segments( Ok(sync_report_from_pairs( video_onsets_s, audio_onsets_s, + activity_start_delta_ms, pairs, )) } @@ -433,6 +437,7 @@ pub(super) fn shortest_wrapped_difference(delta_s: f64, pulse_period_s: f64) -> fn sync_report_from_pairs( video_onsets_s: &[f64], audio_onsets_s: &[f64], + activity_start_delta_ms: f64, pairs: Vec, ) -> SyncAnalysisReport { let paired_events = pairs @@ -466,6 +471,7 @@ fn sync_report_from_pairs( video_event_count: video_onsets_s.len(), audio_event_count: audio_onsets_s.len(), paired_event_count: skews_ms.len(), + activity_start_delta_ms, first_skew_ms, last_skew_ms, mean_skew_ms, diff --git a/client/src/sync_probe/analyze/onset_detection/tests.rs b/client/src/sync_probe/analyze/onset_detection/tests.rs index 32051c9..b807b78 100644 --- a/client/src/sync_probe/analyze/onset_detection/tests.rs +++ b/client/src/sync_probe/analyze/onset_detection/tests.rs @@ -234,6 +234,44 @@ fn correlate_segments_validate_inputs_and_support_single_pulse_fallback() { assert!(correlate_segments(&video, &audio, 1.0, 0.1, 3, 0.05).is_err()); } +#[test] +fn correlate_segments_preserves_whole_period_delay_evidence() { + fn segment(start_s: f64, duration_s: f64) -> PulseSegment { + PulseSegment { + start_s, + end_s: start_s + duration_s, + duration_s, + } + } + + let video = (0..30) + .map(|tick| segment(f64::from(tick), if tick % 5 == 0 { 0.24 } else { 0.12 })) + .collect::>(); + let audio = (0..30) + .map(|tick| { + segment( + f64::from(tick) + 20.0, + if tick % 5 == 0 { 0.24 } else { 0.12 }, + ) + }) + .collect::>(); + + let report = correlate_segments(&video, &audio, 1.0, 0.12, 5, 0.5).expect("correlated report"); + + assert_eq!(report.activity_start_delta_ms, 20_000.0); + assert!( + report.max_abs_skew_ms < 1.0, + "cadence aliasing still creates apparently good pairs" + ); + let verdict = report.verdict(); + assert_eq!(verdict.status, "catastrophic_failure"); + assert!( + verdict.reason.contains("activity starts"), + "unexpected verdict reason: {}", + verdict.reason + ); +} + fn assert_sync_report_shape(report: &SyncAnalysisReport, paired_events: usize) { assert_eq!(report.video_event_count, paired_events); assert_eq!(report.audio_event_count, paired_events); diff --git a/client/src/sync_probe/analyze/report.rs b/client/src/sync_probe/analyze/report.rs index d2a795c..6a0fa7b 100644 --- a/client/src/sync_probe/analyze/report.rs +++ b/client/src/sync_probe/analyze/report.rs @@ -19,6 +19,7 @@ pub struct SyncAnalysisReport { pub video_event_count: usize, pub audio_event_count: usize, pub paired_event_count: usize, + pub activity_start_delta_ms: f64, pub first_skew_ms: f64, pub last_skew_ms: f64, pub mean_skew_ms: f64, @@ -77,6 +78,17 @@ impl SyncAnalysisReport { reason: String::new(), }; + if self.activity_start_delta_ms.abs() >= VERDICT_CATASTROPHIC_MAX_ABS_SKEW_MS { + return SyncAnalysisVerdict { + status: "catastrophic_failure".to_string(), + reason: format!( + "audio/video activity starts are separated by {:+.1} ms, at or above the {:.1} ms catastrophic boundary", + self.activity_start_delta_ms, VERDICT_CATASTROPHIC_MAX_ABS_SKEW_MS + ), + ..base + }; + } + if self.paired_event_count < VERDICT_MIN_PAIRED_EVENTS { return SyncAnalysisVerdict { status: "insufficient_data".to_string(), @@ -146,6 +158,18 @@ impl SyncAnalysisReport { #[must_use] pub fn calibration_recommendation(&self) -> SyncCalibrationRecommendation { + if self.activity_start_delta_ms.abs() >= VERDICT_ACCEPTABLE_P95_ABS_SKEW_MS { + return SyncCalibrationRecommendation { + ready: false, + recommended_audio_offset_adjust_us: 0, + recommended_video_offset_adjust_us: 0, + note: format!( + "activity start delta {:+.1} ms exceeds the {:.1} ms calibration-safe band", + self.activity_start_delta_ms, VERDICT_ACCEPTABLE_P95_ABS_SKEW_MS + ), + }; + } + if self.paired_event_count < CALIBRATION_MIN_PAIRED_EVENTS { return SyncCalibrationRecommendation { ready: false, @@ -247,6 +271,7 @@ mod tests { video_event_count: 4, audio_event_count: 4, paired_event_count: 4, + activity_start_delta_ms: 0.0, first_skew_ms: 20.0, last_skew_ms: 20.0, mean_skew_ms: 20.0, @@ -275,6 +300,7 @@ mod tests { video_event_count: 12, audio_event_count: 12, paired_event_count: 12, + activity_start_delta_ms: 0.0, first_skew_ms: 10.0, last_skew_ms: 70.0, mean_skew_ms: 40.0, @@ -299,6 +325,7 @@ mod tests { video_event_count: 14, audio_event_count: 14, paired_event_count: 12, + activity_start_delta_ms: 0.0, first_skew_ms: 28.0, last_skew_ms: 32.0, mean_skew_ms: 30.0, @@ -324,6 +351,7 @@ mod tests { video_event_count: 14, audio_event_count: 14, paired_event_count: 12, + activity_start_delta_ms: 0.0, first_skew_ms: 3.0, last_skew_ms: 4.0, mean_skew_ms: 3.5, @@ -348,6 +376,7 @@ mod tests { video_event_count: 5, audio_event_count: 5, paired_event_count: 5, + activity_start_delta_ms: 0.0, first_skew_ms: 10.0, last_skew_ms: 20.0, mean_skew_ms: 15.0, @@ -371,6 +400,7 @@ mod tests { video_event_count: 5, audio_event_count: 5, paired_event_count: 5, + activity_start_delta_ms: 0.0, first_skew_ms: 8_000.0, last_skew_ms: 8_000.0, mean_skew_ms: 8_000.0, @@ -387,4 +417,29 @@ mod tests { assert!(!verdict.passed); assert_eq!(verdict.status, "catastrophic_failure"); } + + #[test] + fn verdict_flags_catastrophic_activity_start_delta() { + let report = SyncAnalysisReport { + video_event_count: 20, + audio_event_count: 20, + paired_event_count: 20, + activity_start_delta_ms: 20_000.0, + first_skew_ms: 0.0, + last_skew_ms: 0.0, + mean_skew_ms: 0.0, + median_skew_ms: 0.0, + max_abs_skew_ms: 0.0, + drift_ms: 0.0, + skews_ms: vec![0.0; 20], + video_onsets_s: vec![], + audio_onsets_s: vec![], + paired_events: vec![], + }; + + let verdict = report.verdict(); + assert!(!verdict.passed); + assert_eq!(verdict.status, "catastrophic_failure"); + assert!(verdict.reason.contains("activity starts")); + } } diff --git a/common/Cargo.toml b/common/Cargo.toml index 5a5c5c4..047293e 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.16.11" +version = "0.16.12" edition = "2024" build = "build.rs" diff --git a/scripts/manual/browser_consumer_probe.py b/scripts/manual/browser_consumer_probe.py index 90f912e..5018735 100755 --- a/scripts/manual/browser_consumer_probe.py +++ b/scripts/manual/browser_consumer_probe.py @@ -177,7 +177,10 @@ async function initStream() {{ await postJson('/status', {{ page_message: 'permission granted' }}); const devices = await navigator.mediaDevices.enumerateDevices(); await postJson('/status', {{ page_message: 'devices enumerated', devices: devices.map(fmtDevice) }}); - const videoIn = devices.find(d => d.kind === 'videoinput' && /UGREEN/i.test(d.label)) || devices.find(d => d.kind === 'videoinput'); + const videoIn = + devices.find(d => d.kind === 'videoinput' && /(Lesavka Composite|Multifunction Composite Gadget)/i.test(d.label)) || + devices.find(d => d.kind === 'videoinput' && /UGREEN/i.test(d.label)) || + devices.find(d => d.kind === 'videoinput'); const audioIn = devices.find(d => d.kind === 'audioinput' && /(Multifunction Composite Gadget|Lesavka Composite)/i.test(d.label)) || devices.find(d => d.kind === 'audioinput'); stream = await navigator.mediaDevices.getUserMedia({{ video: videoIn ? {{ deviceId: {{ exact: videoIn.deviceId }} }} : true, diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index 29fc4a6..0414c63 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -887,6 +887,7 @@ lines = [ f"- video onsets: {report['video_event_count']}", f"- audio onsets: {report['audio_event_count']}", f"- paired pulses: {report['paired_event_count']}", + f"- activity start delta: {report.get('activity_start_delta_ms', 0.0):+.1f} ms (audio after video is positive)", f"- first skew: {report['first_skew_ms']:+.1f} ms (audio after video is positive)", f"- last skew: {report['last_skew_ms']:+.1f} ms", f"- mean skew: {report['mean_skew_ms']:+.1f} ms", diff --git a/scripts/manual/run_upstream_browser_av_sync.sh b/scripts/manual/run_upstream_browser_av_sync.sh index 7dcb5ee..24d232d 100755 --- a/scripts/manual/run_upstream_browser_av_sync.sh +++ b/scripts/manual/run_upstream_browser_av_sync.sh @@ -29,6 +29,7 @@ READY_TIMEOUT_SECONDS=${READY_TIMEOUT_SECONDS:-120} mkdir -p "${LOCAL_OUTPUT_DIR}" STAMP="$(date +%Y%m%d-%H%M%S)" LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-browser-av-sync-${STAMP}.webm" +LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR}/lesavka-browser-av-sync-${STAMP}" scp ${SSH_OPTS} "${REPO_ROOT}/scripts/manual/browser_consumer_probe.py" "${TETHYS_HOST}:${REMOTE_SCRIPT}" @@ -162,8 +163,11 @@ scp ${SSH_OPTS} "${TETHYS_HOST}:${REMOTE_CAPTURE}" "${LOCAL_CAPTURE}" echo "==> analyzing browser capture" ( cd "${REPO_ROOT}" - cargo run -p lesavka_client --bin lesavka-sync-analyze -- "${LOCAL_CAPTURE}" + cargo run -p lesavka_client --bin lesavka-sync-analyze -- \ + --report-dir "${LOCAL_REPORT_DIR}" \ + "${LOCAL_CAPTURE}" ) echo "==> done" echo "capture: ${LOCAL_CAPTURE}" +echo "report_dir: ${LOCAL_REPORT_DIR}" diff --git a/server/Cargo.toml b/server/Cargo.toml index 939064c..5a8e580 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.16.11" +version = "0.16.12" edition = "2024" autobins = false