use anyhow::{Context, Result, bail}; use serde::Deserialize; use std::path::Path; use std::process::Command; use super::onset_detection::VideoColorFrame; const VIDEO_ANALYSIS_SIDE_PX: usize = 32; #[derive(Debug, Deserialize)] struct ProbeFrameResponse { #[serde(default)] frames: Vec, } #[derive(Debug, Deserialize)] struct ProbeFrameEntry { best_effort_timestamp_time: Option, } pub(super) fn extract_video_timestamps(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffprobe") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-select_streams") .arg("v:0") .arg("-show_frames") .arg("-show_entries") .arg("frame=best_effort_timestamp_time") .arg("-of") .arg("json") .arg(capture_path), "ffprobe video timestamps", )?; let response: ProbeFrameResponse = serde_json::from_slice(&output).context("parsing ffprobe frame JSON")?; let timestamps = response .frames .into_iter() .filter_map(|entry| entry.best_effort_timestamp_time) .map(|value| value.parse::().context("parsing frame timestamp")) .collect::>>()?; if timestamps.is_empty() { bail!("ffprobe did not return any video frame timestamps"); } Ok(timestamps) } pub(super) fn extract_video_brightness(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffmpeg") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-i") .arg(capture_path) .arg("-map") .arg("0:v:0") .arg("-vf") .arg(format!( "scale={side}:{side}:flags=area,format=gray", side = VIDEO_ANALYSIS_SIDE_PX )) .arg("-f") .arg("rawvideo") .arg("-pix_fmt") .arg("gray") .arg("-"), "ffmpeg video brightness extraction", )?; if output.is_empty() { bail!("ffmpeg did not emit any video brightness data"); } let frame_pixels = VIDEO_ANALYSIS_SIDE_PX * VIDEO_ANALYSIS_SIDE_PX; if output.len() % frame_pixels != 0 { bail!( "ffmpeg emitted {} bytes of video brightness data, which is not divisible by the {}-pixel analysis frame size", output.len(), frame_pixels ); } let extracted_frames = output.len() / frame_pixels; Ok(output .chunks_exact(frame_pixels) .take(extracted_frames) .map(summarize_frame_brightness) .collect()) } pub(super) fn extract_video_colors(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffmpeg") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-i") .arg(capture_path) .arg("-map") .arg("0:v:0") .arg("-vf") .arg(format!( "scale={side}:{side}:flags=area,format=rgb24", side = VIDEO_ANALYSIS_SIDE_PX )) .arg("-f") .arg("rawvideo") .arg("-pix_fmt") .arg("rgb24") .arg("-"), "ffmpeg video color extraction", )?; if output.is_empty() { bail!("ffmpeg did not emit any video color data"); } let frame_bytes = VIDEO_ANALYSIS_SIDE_PX * VIDEO_ANALYSIS_SIDE_PX * 3; if output.len() % frame_bytes != 0 { bail!( "ffmpeg emitted {} bytes of video color data, which is not divisible by the {}-byte analysis frame size", output.len(), frame_bytes ); } let extracted_frames = output.len() / frame_bytes; Ok(output .chunks_exact(frame_bytes) .take(extracted_frames) .map(summarize_frame_color) .collect()) } pub(super) fn extract_audio_samples(capture_path: &Path) -> Result> { let output = run_command( Command::new("ffmpeg") .arg("-hide_banner") .arg("-loglevel") .arg("error") .arg("-i") .arg(capture_path) .arg("-map") .arg("0:a:0") .arg("-ac") .arg("1") .arg("-ar") .arg(super::onset_detection::DEFAULT_AUDIO_SAMPLE_RATE_HZ.to_string()) .arg("-f") .arg("s16le") .arg("-acodec") .arg("pcm_s16le") .arg("-"), "ffmpeg audio extraction", )?; if output.len() < 2 { bail!("ffmpeg did not emit enough audio data to analyze"); } Ok(output .chunks_exact(2) .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) .collect()) } pub(super) fn run_command(command: &mut Command, description: &str) -> Result> { let output = command .output() .with_context(|| format!("running {description}"))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); bail!("{description} failed: {}", stderr.trim()); } Ok(output.stdout) } fn summarize_frame_brightness(frame: &[u8]) -> u8 { let mean = frame.iter().map(|value| u64::from(*value)).sum::() / frame.len().max(1) as u64; mean.min(u64::from(u8::MAX)) as u8 } fn summarize_frame_color(frame: &[u8]) -> VideoColorFrame { let mut r_sum = 0u64; let mut g_sum = 0u64; let mut b_sum = 0u64; let mut selected = 0u64; for pixel in frame.chunks_exact(3) { let r = pixel[0]; let g = pixel[1]; let b = pixel[2]; let max = r.max(g).max(b); let min = r.min(g).min(b); if max >= 60 && max.saturating_sub(min) >= 24 { r_sum += u64::from(r); g_sum += u64::from(g); b_sum += u64::from(b); selected += 1; } } if selected == 0 { selected = (frame.len() / 3).max(1) as u64; for pixel in frame.chunks_exact(3) { r_sum += u64::from(pixel[0]); g_sum += u64::from(pixel[1]); b_sum += u64::from(pixel[2]); } } VideoColorFrame { r: (r_sum / selected).min(u64::from(u8::MAX)) as u8, g: (g_sum / selected).min(u64::from(u8::MAX)) as u8, b: (b_sum / selected).min(u64::from(u8::MAX)) as u8, } } #[cfg(test)] mod tests { use super::{ extract_audio_samples, extract_video_brightness, extract_video_colors, extract_video_timestamps, run_command, }; use crate::sync_probe::analyze::test_support::{ audio_samples_to_bytes, frame_json, thumbnail_rgb_video_bytes, thumbnail_video_bytes, with_fake_media_tools, }; use std::process::Command; #[test] fn extract_video_timestamps_reads_fake_ffprobe_output() { let timestamps = vec![0.0, 0.5, 1.0]; with_fake_media_tools( &frame_json(×tamps), &[1, 2, 3], &[1, 0], |capture_path| { let parsed = extract_video_timestamps(capture_path).expect("video timestamps"); assert_eq!(parsed, timestamps); }, ); } #[test] fn extract_video_timestamps_rejects_empty_and_invalid_outputs() { with_fake_media_tools(br#"{"frames":[]}"#, &[1], &[1, 0], |capture_path| { let error = extract_video_timestamps(capture_path).expect_err("empty frames fail"); assert!( error .to_string() .contains("did not return any video frame timestamps") ); }); with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"bad"}]}"#, &[1], &[1, 0], |capture_path| { let error = extract_video_timestamps(capture_path).expect_err("invalid timestamp fails"); assert!(error.to_string().contains("parsing frame timestamp")); }, ); } #[test] fn extract_video_brightness_reads_fake_ffmpeg_output() { let brightness = vec![5u8, 100, 250]; with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &thumbnail_video_bytes(&brightness), &[1, 0], |capture_path| { let parsed = extract_video_brightness(capture_path).expect("video brightness"); assert_eq!(parsed, vec![16, 40, 77]); }, ); } #[test] fn extract_video_brightness_rejects_empty_output() { with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[], &[1, 0], |capture_path| { let error = extract_video_brightness(capture_path).expect_err("empty brightness"); assert!( error .to_string() .contains("did not emit any video brightness data") ); }, ); } #[test] fn extract_video_brightness_uses_full_frame_thumbnail_average() { let brightness = vec![20u8, 45, 20]; with_fake_media_tools( &frame_json(&[0.0, 0.1, 0.2]), &thumbnail_video_bytes(&brightness), &[1, 0], |capture_path| { let parsed = extract_video_brightness(capture_path).expect("video brightness"); assert_eq!(parsed, vec![20, 26, 20]); }, ); } #[test] fn extract_video_brightness_rejects_truncated_frame_data() { with_fake_media_tools(&frame_json(&[0.0]), &[1, 2, 3], &[1, 0], |capture_path| { let error = extract_video_brightness(capture_path).expect_err("truncated frame bytes"); assert!(error.to_string().contains("not divisible")); }); } #[test] fn extract_video_colors_reads_fake_ffmpeg_output() { let colors = vec![(255, 45, 45), (0, 230, 118), (41, 121, 255)]; with_fake_media_tools( &frame_json(&[0.0, 0.1, 0.2]), &thumbnail_rgb_video_bytes(&colors), &[1, 0], |capture_path| { let parsed = extract_video_colors(capture_path).expect("video colors"); assert_eq!(parsed[0].r, 255); assert_eq!(parsed[1].g, 230); assert_eq!(parsed[2].b, 255); }, ); } #[test] fn extract_audio_samples_reads_fake_ffmpeg_output() { let samples = vec![1i16, -2, 32_000]; with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[1], &audio_samples_to_bytes(&samples), |capture_path| { let parsed = extract_audio_samples(capture_path).expect("audio samples"); assert_eq!(parsed, samples); }, ); } #[test] fn extract_audio_samples_rejects_too_short_output() { with_fake_media_tools( br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#, &[1], &[7], |capture_path| { let error = extract_audio_samples(capture_path).expect_err("short audio"); assert!( error .to_string() .contains("did not emit enough audio data to analyze") ); }, ); } #[test] fn run_command_reports_success_and_failure() { let output = run_command( Command::new("sh").arg("-c").arg("printf 'ok'"), "success command", ) .expect("success output"); assert_eq!(output, b"ok"); let error = run_command( Command::new("sh") .arg("-c") .arg("printf 'boom' >&2; exit 7"), "failing command", ) .expect_err("failing command should error"); assert!(error.to_string().contains("failing command failed: boom")); } }