lesavka/client/src/sync_probe/analyze.rs

//! Analyze captured upstream sync-probe media for audio/video skew and drift.

mod media_extract;
mod onset_detection;
mod report;
#[cfg(test)]
pub(super) mod test_support;

use anyhow::{Result, bail};
use std::path::Path;

use media_extract::{extract_audio_samples, extract_video_brightness, extract_video_timestamps};
use onset_detection::{
    DEFAULT_AUDIO_SAMPLE_RATE_HZ, correlate_segments, detect_audio_segments, detect_video_segments,
};

pub use onset_detection::{detect_audio_onsets, detect_video_onsets};
pub use report::{
    SyncAnalysisOptions, SyncAnalysisReport, SyncAnalysisVerdict, SyncCalibrationRecommendation,
    SyncEventPair,
};

/// Analyzes a captured upstream sync-probe file by extracting video and audio
/// pulses, then correlating them into skew and drift metrics.
pub fn analyze_capture(
    capture_path: &Path,
    options: &SyncAnalysisOptions,
) -> Result<SyncAnalysisReport> {
    let raw_timestamps = extract_video_timestamps(capture_path)?;
    let brightness = extract_video_brightness(capture_path)?;
    let timestamps = reconcile_video_timestamps(raw_timestamps, brightness.len())?;
    let video_segments = detect_video_segments(&timestamps, &brightness)?;

    let audio_samples = extract_audio_samples(capture_path)?;
    let audio_segments = detect_audio_segments(
        &audio_samples,
        DEFAULT_AUDIO_SAMPLE_RATE_HZ,
        options.audio_window_ms,
    )?;

    correlate_segments(
        &video_segments,
        &audio_segments,
        options.pulse_period_s,
        options.pulse_width_s,
        options.marker_tick_period,
        options.max_pair_gap_s,
    )
}

fn reconcile_video_timestamps(timestamps: Vec<f64>, frame_count: usize) -> Result<Vec<f64>> {
    if frame_count == 0 {
        bail!("capture did not contain any decoded video brightness frames");
    }
    if timestamps.len() == frame_count {
        return Ok(timestamps);
    }

    let first = timestamps.first().copied();
    let last = timestamps.last().copied();
    if let (Some(first), Some(last)) = (first, last)
        && frame_count > 1
        && last > first
    {
        let step = (last - first) / (frame_count - 1) as f64;
        return Ok((0..frame_count)
            .map(|index| first + index as f64 * step)
            .collect());
    }

    if timestamps.len() > frame_count {
        return Ok(timestamps.into_iter().take(frame_count).collect());
    }

    bail!(
        "ffprobe returned {} video timestamps for {frame_count} decoded brightness frames and no usable duration",
        timestamps.len()
    )
}

#[cfg(test)]
mod tests {
    use super::test_support::{
        audio_samples_to_bytes, click_track_samples, frame_json, thumbnail_video_bytes,
        with_fake_media_tools,
    };
    use super::{SyncAnalysisOptions, analyze_capture};
    use crate::sync_probe::analyze::reconcile_video_timestamps;

    #[test]
    fn analyze_capture_runs_against_fake_media_tools() {
        let timestamps = (0..15).map(|index| index as f64 / 10.0).collect::<Vec<_>>();
        let brightness = timestamps
            .iter()
            .enumerate()
            .map(|(index, _)| if matches!(index, 0 | 5 | 10) { 250 } else { 5 })
            .collect::<Vec<_>>();
        let audio = click_track_samples(&[0.05, 0.55, 1.05], 53_000);

        with_fake_media_tools(
            &frame_json(&timestamps),
            &thumbnail_video_bytes(&brightness),
            &audio_samples_to_bytes(&audio),
            |capture_path| {
                let report = analyze_capture(
                    capture_path,
                    &SyncAnalysisOptions {
                        pulse_period_s: 0.5,
                        ..SyncAnalysisOptions::default()
                    },
                )
                .expect("analysis report");
                assert_eq!(report.video_event_count, 3);
                assert_eq!(report.audio_event_count, 3);
                assert_eq!(report.paired_event_count, 3);
                assert_eq!(report.skews_ms.len(), 3);
                assert!((report.first_skew_ms - 50.0).abs() < 10.0);
                assert!(report.max_abs_skew_ms < 120.0);
            },
        );
    }

    #[test]
    fn analyze_capture_synthesizes_timestamps_when_mjpeg_metadata_overreports_frames() {
        let metadata_timestamps = (0..301)
            .map(|index| index as f64 * 0.004)
            .collect::<Vec<_>>();
        let brightness = (0..25)
            .map(|index| if matches!(index, 0 | 10 | 20) { 250 } else { 5 })
            .collect::<Vec<_>>();
        let audio = click_track_samples(&[0.05, 0.55, 1.05], 67_000);

        with_fake_media_tools(
            &frame_json(&metadata_timestamps),
            &thumbnail_video_bytes(&brightness),
            &audio_samples_to_bytes(&audio),
            |capture_path| {
                let report = analyze_capture(
                    capture_path,
                    &SyncAnalysisOptions {
                        pulse_period_s: 0.5,
                        ..SyncAnalysisOptions::default()
                    },
                )
                .expect("analysis report");
                assert_eq!(report.video_event_count, 3);
                assert_eq!(report.audio_event_count, 3);
                assert_eq!(report.paired_event_count, 3);
                assert!(report.max_abs_skew_ms < 120.0);
            },
        );
    }

    #[test]
    fn reconcile_video_timestamps_resamples_metadata_span_to_decoded_frame_count() {
        let reconciled = reconcile_video_timestamps(vec![0.0, 0.004, 0.008, 1.0], 3)
            .expect("reconciled timestamps");
        assert_eq!(reconciled, vec![0.0, 0.5, 1.0]);
    }
}