lesavka/client/src/sync_probe/analyze/media_extract.rs

use anyhow::{Context, Result, bail};
use serde::Deserialize;
use std::path::Path;
use std::process::Command;

const VIDEO_ANALYSIS_SIDE_PX: usize = 32;

#[derive(Debug, Deserialize)]
struct ProbeFrameResponse {
    #[serde(default)]
    frames: Vec<ProbeFrameEntry>,
}

#[derive(Debug, Deserialize)]
struct ProbeFrameEntry {
    best_effort_timestamp_time: Option<String>,
}

pub(super) fn extract_video_timestamps(capture_path: &Path) -> Result<Vec<f64>> {
    let output = run_command(
        Command::new("ffprobe")
            .arg("-hide_banner")
            .arg("-loglevel")
            .arg("error")
            .arg("-select_streams")
            .arg("v:0")
            .arg("-show_frames")
            .arg("-show_entries")
            .arg("frame=best_effort_timestamp_time")
            .arg("-of")
            .arg("json")
            .arg(capture_path),
        "ffprobe video timestamps",
    )?;
    let response: ProbeFrameResponse =
        serde_json::from_slice(&output).context("parsing ffprobe frame JSON")?;
    let timestamps = response
        .frames
        .into_iter()
        .filter_map(|entry| entry.best_effort_timestamp_time)
        .map(|value| value.parse::<f64>().context("parsing frame timestamp"))
        .collect::<Result<Vec<_>>>()?;
    if timestamps.is_empty() {
        bail!("ffprobe did not return any video frame timestamps");
    }
    Ok(timestamps)
}

pub(super) fn extract_video_brightness(capture_path: &Path) -> Result<Vec<u8>> {
    let output = run_command(
        Command::new("ffmpeg")
            .arg("-hide_banner")
            .arg("-loglevel")
            .arg("error")
            .arg("-i")
            .arg(capture_path)
            .arg("-map")
            .arg("0:v:0")
            .arg("-vf")
            .arg(format!(
                "scale={side}:{side}:flags=area,format=gray",
                side = VIDEO_ANALYSIS_SIDE_PX
            ))
            .arg("-f")
            .arg("rawvideo")
            .arg("-pix_fmt")
            .arg("gray")
            .arg("-"),
        "ffmpeg video brightness extraction",
    )?;
    if output.is_empty() {
        bail!("ffmpeg did not emit any video brightness data");
    }

    let frame_pixels = VIDEO_ANALYSIS_SIDE_PX * VIDEO_ANALYSIS_SIDE_PX;
    if output.len() % frame_pixels != 0 {
        bail!(
            "ffmpeg emitted {} bytes of video brightness data, which is not divisible by the {}-pixel analysis frame size",
            output.len(),
            frame_pixels
        );
    }
    let extracted_frames = output.len() / frame_pixels;

    Ok(output
        .chunks_exact(frame_pixels)
        .take(extracted_frames)
        .map(summarize_frame_brightness)
        .collect())
}

pub(super) fn extract_audio_samples(capture_path: &Path) -> Result<Vec<i16>> {
    let output = run_command(
        Command::new("ffmpeg")
            .arg("-hide_banner")
            .arg("-loglevel")
            .arg("error")
            .arg("-i")
            .arg(capture_path)
            .arg("-map")
            .arg("0:a:0")
            .arg("-ac")
            .arg("1")
            .arg("-ar")
            .arg(super::onset_detection::DEFAULT_AUDIO_SAMPLE_RATE_HZ.to_string())
            .arg("-f")
            .arg("s16le")
            .arg("-acodec")
            .arg("pcm_s16le")
            .arg("-"),
        "ffmpeg audio extraction",
    )?;
    if output.len() < 2 {
        bail!("ffmpeg did not emit enough audio data to analyze");
    }
    Ok(output
        .chunks_exact(2)
        .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
        .collect())
}

pub(super) fn run_command(command: &mut Command, description: &str) -> Result<Vec<u8>> {
    let output = command
        .output()
        .with_context(|| format!("running {description}"))?;
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        bail!("{description} failed: {}", stderr.trim());
    }
    Ok(output.stdout)
}

fn summarize_frame_brightness(frame: &[u8]) -> u8 {
    let mean = frame.iter().map(|value| u64::from(*value)).sum::<u64>() / frame.len().max(1) as u64;
    mean.min(u64::from(u8::MAX)) as u8
}

#[cfg(test)]
mod tests {
    use super::{
        extract_audio_samples, extract_video_brightness, extract_video_timestamps, run_command,
    };
    use crate::sync_probe::analyze::test_support::{
        audio_samples_to_bytes, frame_json, thumbnail_video_bytes, with_fake_media_tools,
    };
    use std::process::Command;

    #[test]
    fn extract_video_timestamps_reads_fake_ffprobe_output() {
        let timestamps = vec![0.0, 0.5, 1.0];
        with_fake_media_tools(
            &frame_json(&timestamps),
            &[1, 2, 3],
            &[1, 0],
            |capture_path| {
                let parsed = extract_video_timestamps(capture_path).expect("video timestamps");
                assert_eq!(parsed, timestamps);
            },
        );
    }

    #[test]
    fn extract_video_timestamps_rejects_empty_and_invalid_outputs() {
        with_fake_media_tools(br#"{"frames":[]}"#, &[1], &[1, 0], |capture_path| {
            let error = extract_video_timestamps(capture_path).expect_err("empty frames fail");
            assert!(
                error
                    .to_string()
                    .contains("did not return any video frame timestamps")
            );
        });

        with_fake_media_tools(
            br#"{"frames":[{"best_effort_timestamp_time":"bad"}]}"#,
            &[1],
            &[1, 0],
            |capture_path| {
                let error =
                    extract_video_timestamps(capture_path).expect_err("invalid timestamp fails");
                assert!(error.to_string().contains("parsing frame timestamp"));
            },
        );
    }

    #[test]
    fn extract_video_brightness_reads_fake_ffmpeg_output() {
        let brightness = vec![5u8, 100, 250];
        with_fake_media_tools(
            br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#,
            &thumbnail_video_bytes(&brightness),
            &[1, 0],
            |capture_path| {
                let parsed = extract_video_brightness(capture_path).expect("video brightness");
                assert_eq!(parsed, vec![16, 40, 77]);
            },
        );
    }

    #[test]
    fn extract_video_brightness_rejects_empty_output() {
        with_fake_media_tools(
            br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#,
            &[],
            &[1, 0],
            |capture_path| {
                let error = extract_video_brightness(capture_path).expect_err("empty brightness");
                assert!(
                    error
                        .to_string()
                        .contains("did not emit any video brightness data")
                );
            },
        );
    }

    #[test]
    fn extract_video_brightness_uses_full_frame_thumbnail_average() {
        let brightness = vec![20u8, 45, 20];
        with_fake_media_tools(
            &frame_json(&[0.0, 0.1, 0.2]),
            &thumbnail_video_bytes(&brightness),
            &[1, 0],
            |capture_path| {
                let parsed = extract_video_brightness(capture_path).expect("video brightness");
                assert_eq!(parsed, vec![20, 26, 20]);
            },
        );
    }

    #[test]
    fn extract_video_brightness_rejects_truncated_frame_data() {
        with_fake_media_tools(&frame_json(&[0.0]), &[1, 2, 3], &[1, 0], |capture_path| {
            let error = extract_video_brightness(capture_path).expect_err("truncated frame bytes");
            assert!(error.to_string().contains("not divisible"));
        });
    }

    #[test]
    fn extract_audio_samples_reads_fake_ffmpeg_output() {
        let samples = vec![1i16, -2, 32_000];
        with_fake_media_tools(
            br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#,
            &[1],
            &audio_samples_to_bytes(&samples),
            |capture_path| {
                let parsed = extract_audio_samples(capture_path).expect("audio samples");
                assert_eq!(parsed, samples);
            },
        );
    }

    #[test]
    fn extract_audio_samples_rejects_too_short_output() {
        with_fake_media_tools(
            br#"{"frames":[{"best_effort_timestamp_time":"0.0"}]}"#,
            &[1],
            &[7],
            |capture_path| {
                let error = extract_audio_samples(capture_path).expect_err("short audio");
                assert!(
                    error
                        .to_string()
                        .contains("did not emit enough audio data to analyze")
                );
            },
        );
    }

    #[test]
    fn run_command_reports_success_and_failure() {
        let output = run_command(
            Command::new("sh").arg("-c").arg("printf 'ok'"),
            "success command",
        )
        .expect("success output");
        assert_eq!(output, b"ok");

        let error = run_command(
            Command::new("sh")
                .arg("-c")
                .arg("printf 'boom' >&2; exit 7"),
            "failing command",
        )
        .expect_err("failing command should error");
        assert!(error.to_string().contains("failing command failed: boom"));
    }
}