use anyhow::{Context, Result, bail}; use serde::Deserialize; use std::path::Path; use std::process::Command; const VIDEO_ANALYSIS_SIDE_PX: usize = 32; #[derive(Debug, Deserialize)] struct ProbeFrameResponse { #[serde(default)] frames: Vec, } #[derive(Debug, Deserialize)] struct ProbeFrameEntry { best_effort_timestamp_time: Option, } pub(super) fn extract_video_timestamps(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffprobe") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-select_streams") .arg("v:0") .arg("-show_frames") .arg("-show_entries") .arg("frame=best_effort_timestamp_time") .arg("-of") .arg("json") .arg(capture_path), "ffprobe video timestamps", )?; let response: ProbeFrameResponse = serde_json::from_slice(&output).context("parsing ffprobe frame JSON")?; let timestamps = response .frames .into_iter() .filter_map(|entry| entry.best_effort_timestamp_time) .map(|value| value.parse::().context("parsing frame timestamp")) .collect::>>()?; if timestamps.is_empty() { bail!("ffprobe did not return any video frame timestamps"); } Ok(timestamps) } pub(super) fn extract_video_brightness( capture_path: &Path, expected_frames: usize, ) -> Result> { if expected_frames == 0 { bail!("expected at least one video frame when extracting brightness"); } let output = run_command( Command::new("ffmpeg") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-i") .arg(capture_path) .arg("-map") .arg("0:v:0") .arg("-vf") .arg(format!( "scale={side}:{side}:flags=area,format=gray", side = VIDEO_ANALYSIS_SIDE_PX )) .arg("-f") .arg("rawvideo") .arg("-pix_fmt") .arg("gray") .arg("-"), "ffmpeg video brightness extraction", )?; if output.is_empty() { bail!("ffmpeg did not emit any video brightness data"); } let frame_pixels = VIDEO_ANALYSIS_SIDE_PX * VIDEO_ANALYSIS_SIDE_PX; if output.len() % frame_pixels != 0 { bail!( "ffmpeg emitted {} bytes of video brightness data, which is not divisible by the {}-pixel analysis frame size", output.len(), frame_pixels ); } let extracted_frames = output.len() / frame_pixels; if extracted_frames < expected_frames { bail!( "ffmpeg emitted only {extracted_frames} brightness frames for {expected_frames} expected timestamps" ); } Ok(output .chunks_exact(frame_pixels) .take(expected_frames) .map(summarize_frame_brightness) .collect()) } pub(super) fn extract_audio_samples(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffmpeg") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-i") .arg(capture_path) .arg("-map") .arg("0:a:0") .arg("-ac") .arg("1") .arg("-ar") .arg(super::onset_detection::DEFAULT_AUDIO_SAMPLE_RATE_HZ.to_string()) .arg("-f") .arg("s16le") .arg("-acodec") .arg("pcm_s16le") .arg("-"), "ffmpeg audio extraction", )?; if output.len() < 2 { bail!("ffmpeg did not emit enough audio data to analyze"); } Ok(output .chunks_exact(2) .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) .collect()) } pub(super) fn run_command(command: &mut Command, description: &str) -> Result> { let output = command .output() .with_context(|| format!("running {description}"))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); bail!("{description} failed: {}", stderr.trim()); } Ok(output.stdout) } fn summarize_frame_brightness(frame: &[u8]) -> u8 { let mean = frame.iter().map(|value| u64::from(*value)).sum::() / frame.len().max(1) as u64; mean.min(u64::from(u8::MAX)) as u8 } #[cfg(test)] mod tests { use super::{ extract_audio_samples, extract_video_brightness, extract_video_timestamps, run_command, }; use crate::sync_probe::analyze::test_support::{ audio_samples_to_bytes, frame_json, thumbnail_video_bytes, with_fake_media_tools, }; use std::process::Command; #[test] fn extract_video_timestamps_reads_fake_ffprobe_output() { let timestamps = vec![0.0, 0.5, 1.0]; with_fake_media_tools( &frame_json(×tamps), &[1, 2, 3], &[1, 0], |capture_path| { let parsed = extract_video_timestamps(capture_path).expect("video timestamps"); assert_eq!(parsed, timestamps); }, ); } #[test] fn extract_video_timestamps_rejects_empty_and_invalid_outputs() { with_fake_media_tools(br#"{"frames":[]}"#, &[1], &[1, 0], |capture_path| { let error = extract_video_timestamps(capture_path).expect_err("empty frames fail"); assert!( error .to_string() .contains("did not return any video frame timestamps") ); }); with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"bad"}]}"#, &[1], &[1, 0], |capture_path| { let error = extract_video_timestamps(capture_path).expect_err("invalid timestamp fails"); assert!(error.to_string().contains("parsing frame timestamp")); }, ); } #[test] fn extract_video_brightness_reads_fake_ffmpeg_output() { let brightness = vec![5u8, 100, 250]; with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &thumbnail_video_bytes(&brightness), &[1, 0], |capture_path| { let parsed = extract_video_brightness(capture_path, 1).expect("video brightness"); assert_eq!(parsed, vec![16]); }, ); } #[test] fn extract_video_brightness_rejects_empty_output() { with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[], &[1, 0], |capture_path| { let error = extract_video_brightness(capture_path, 1).expect_err("empty brightness"); assert!( error .to_string() .contains("did not emit any video brightness data") ); }, ); } #[test] fn extract_video_brightness_uses_full_frame_thumbnail_average() { let brightness = vec![20u8, 45, 20]; with_fake_media_tools( &frame_json(&[0.0, 0.1, 0.2]), &thumbnail_video_bytes(&brightness), &[1, 0], |capture_path| { let parsed = extract_video_brightness(capture_path, 3).expect("video brightness"); assert_eq!(parsed, vec![20, 26, 20]); }, ); } #[test] fn extract_video_brightness_rejects_truncated_frame_data() { with_fake_media_tools(&frame_json(&[0.0]), &[1, 2, 3], &[1, 0], |capture_path| { let error = extract_video_brightness(capture_path, 1).expect_err("truncated frame bytes"); assert!(error.to_string().contains("not divisible")); }); } #[test] fn extract_audio_samples_reads_fake_ffmpeg_output() { let samples = vec![1i16, -2, 32_000]; with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[1], &audio_samples_to_bytes(&samples), |capture_path| { let parsed = extract_audio_samples(capture_path).expect("audio samples"); assert_eq!(parsed, samples); }, ); } #[test] fn extract_audio_samples_rejects_too_short_output() { with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[1], &[7], |capture_path| { let error = extract_audio_samples(capture_path).expect_err("short audio"); assert!( error .to_string() .contains("did not emit enough audio data to analyze") ); }, ); } #[test] fn run_command_reports_success_and_failure() { let output = run_command( Command::new("sh").arg("-c").arg("printf 'ok'"), "success command", ) .expect("success output"); assert_eq!(output, b"ok"); let error = run_command( Command::new("sh") .arg("-c") .arg("printf 'boom' >&2; exit 7"), "failing command", ) .expect_err("failing command should error"); assert!(error.to_string().contains("failing command failed: boom")); } }