diff --git a/Cargo.lock b/Cargo.lock index 3f26725..eafc616 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.18" +version = "0.19.19" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.18" +version = "0.19.19" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.18" +version = "0.19.19" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 2919e7b..bdc5a22 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.18" +version = "0.19.19" edition = "2024" [dependencies] diff --git a/client/src/bin/lesavka-sync-analyze.rs b/client/src/bin/lesavka-sync-analyze.rs index ea3efa7..9a70763 100644 --- a/client/src/bin/lesavka-sync-analyze.rs +++ b/client/src/bin/lesavka-sync-analyze.rs @@ -481,6 +481,8 @@ mod tests { audio_onsets_s: vec![9.135, 11.146, 13.135], paired_events: vec![SyncEventPair { event_id: 0, + server_event_id: None, + event_code: None, video_time_s: 11.420, audio_time_s: 11.146, skew_ms: -188.4, diff --git a/client/src/sync_probe/analyze.rs b/client/src/sync_probe/analyze.rs index c99b5aa..1310fd2 100644 --- a/client/src/sync_probe/analyze.rs +++ b/client/src/sync_probe/analyze.rs @@ -249,17 +249,14 @@ mod tests { .enumerate() .map(|(index, _)| match index { 10 | 11 => (255, 45, 45), - 20 | 21 | 22 | 23 => (0, 230, 118), - 30 | 31 | 32 => (41, 121, 255), + 20 | 21 => (0, 230, 118), + 30 | 31 => (41, 121, 255), _ => (0, 0, 0), }) .collect::>(); let mut audio = vec![0i16; 220_000]; - for (start_s, code) in [(1.05, 1usize), (2.05, 2usize), (3.05, 3usize)] { - let start = (start_s * 48_000.0) as usize; - for sample in audio.iter_mut().skip(start).take(5_760 * code) { - *sample = 18_000; - } + for (start_s, frequency_hz) in [(1.05, 620.0), (2.05, 780.0), (3.05, 940.0)] { + add_sine(&mut audio, 48_000, start_s, 0.12, frequency_hz, 18_000.0); } with_fake_media_tools( @@ -289,4 +286,23 @@ mod tests { .expect("reconciled timestamps"); assert_eq!(reconciled, vec![0.0, 0.5, 1.0]); } + + fn add_sine( + samples: &mut [i16], + sample_rate_hz: u32, + start_s: f64, + duration_s: f64, + frequency_hz: f64, + amplitude: f64, + ) { + let start = (start_s * f64::from(sample_rate_hz)).round() as usize; + let len = (duration_s * f64::from(sample_rate_hz)).round() as usize; + for (offset, sample) in samples.iter_mut().skip(start).take(len).enumerate() { + let t = offset as f64 / f64::from(sample_rate_hz); + let value = amplitude * (2.0 * std::f64::consts::PI * frequency_hz * t).sin(); + *sample = value + .round() + .clamp(f64::from(i16::MIN), f64::from(i16::MAX)) as i16; + } + } } diff --git a/client/src/sync_probe/analyze/onset_detection.rs b/client/src/sync_probe/analyze/onset_detection.rs index 2992b22..036637a 100644 --- a/client/src/sync_probe/analyze/onset_detection.rs +++ b/client/src/sync_probe/analyze/onset_detection.rs @@ -23,7 +23,10 @@ const MAX_AUDIO_PULSE_INTERNAL_GAP_S: f64 = 0.16; const MIN_AUDIO_PROBE_PEAK: f64 = 25.0; const AUDIO_ENVELOPE_THRESHOLD_FRACTION: f64 = 0.30; const AUDIO_SAMPLE_THRESHOLD_FRACTION: f64 = 0.22; -const AUDIO_TONE_FREQUENCIES_HZ: [f64; 4] = [660.0, 880.0, 1100.0, 1320.0]; +const AUDIO_TONE_FREQUENCIES_HZ: [f64; 16] = [ + 620.0, 780.0, 940.0, 1120.0, 1320.0, 1540.0, 1780.0, 2040.0, 2320.0, 2620.0, 2960.0, 3340.0, + 3760.0, 4220.0, 4740.0, 5320.0, +]; const MIN_TONE_ENVELOPE_PEAK: f64 = 18.0; const MIN_TONE_CONTRAST_FRACTION_OF_AMPLITUDE: f64 = 0.12; const MIN_TONE_CODE_DOMINANCE_RATIO: f64 = 1.35; @@ -147,7 +150,6 @@ pub(crate) fn detect_color_coded_video_segments( } let frame_step_s = median_frame_step_seconds(×tamps_s[..frame_count]).max(1.0 / 120.0); - let max_event_code = event_codes.iter().copied().max().unwrap_or(1); let mut segments = Vec::new(); let mut previous_code = None::; let mut segment_start = 0.0_f64; @@ -156,7 +158,7 @@ pub(crate) fn detect_color_coded_video_segments( let mut segment_codes = Vec::::new(); for (timestamp, frame) in timestamps_s.iter().copied().zip(frames.iter().copied()) { - let code = color_event_code(frame).filter(|code| event_codes.contains(code)); + let code = color_event_code_for_codes(frame, event_codes); if code.is_some() && previous_code.is_none() { segment_start = previous_timestamp .map(|prior| edge_midpoint(prior, timestamp)) @@ -176,7 +178,6 @@ pub(crate) fn detect_color_coded_video_segments( timestamp, ), pulse_width_s, - max_event_code, &segment_codes, frame_step_s, ); @@ -192,7 +193,6 @@ pub(crate) fn detect_color_coded_video_segments( segment_start, last_timestamp + frame_step_s / 2.0, pulse_width_s, - max_event_code, &segment_codes, frame_step_s, ); @@ -224,7 +224,6 @@ fn push_color_segment( start_s: f64, observed_end_s: f64, pulse_width_s: f64, - max_event_code: u32, codes: &[u32], frame_step_s: f64, ) { @@ -232,9 +231,8 @@ fn push_color_segment( return; }; let observed_duration_s = observed_end_s - start_s; - let max_observed_duration_s = - (pulse_width_s * f64::from(max_event_code) * MAX_COLOR_OBSERVED_DURATION_MULTIPLIER) - + MAX_COLOR_OBSERVED_DURATION_SLACK_S; + let max_observed_duration_s = (pulse_width_s * MAX_COLOR_OBSERVED_DURATION_MULTIPLIER) + + MAX_COLOR_OBSERVED_DURATION_SLACK_S; if observed_duration_s > max_observed_duration_s { return; } @@ -261,19 +259,22 @@ fn dominant_event_code(codes: &[u32]) -> Option { .map(|(code, _)| code) } -fn color_event_code(frame: VideoColorFrame) -> Option { +fn color_event_code_for_codes(frame: VideoColorFrame, event_codes: &[u32]) -> Option { let max = frame.r.max(frame.g).max(frame.b); let min = frame.r.min(frame.g).min(frame.b); if max < MIN_COLOR_PULSE_VALUE || max.saturating_sub(min) < MIN_COLOR_PULSE_SATURATION { return None; } - color_palette() - .into_iter() - .map(|(code, color)| (code, color_distance_squared(frame, color))) + event_codes + .iter() + .copied() + .filter_map(|code| { + color_for_event_code(code).map(|color| (code, color_distance_squared(frame, color))) + }) .min_by_key(|(_, distance)| *distance) .and_then(|(code, distance)| (distance <= MAX_COLOR_DISTANCE_SQUARED).then_some(code)) - .or_else(|| dominant_color_event_code(frame)) + .or_else(|| dominant_color_event_code(frame).filter(|code| event_codes.contains(code))) } fn dominant_color_event_code(frame: VideoColorFrame) -> Option { @@ -305,7 +306,7 @@ fn color_for_event_code(code: u32) -> Option { .find_map(|(palette_code, color)| (palette_code == code).then_some(color)) } -fn color_palette() -> [(u32, VideoColorFrame); 4] { +fn color_palette() -> [(u32, VideoColorFrame); 16] { [ ( 1, @@ -339,6 +340,102 @@ fn color_palette() -> [(u32, VideoColorFrame); 4] { b: 0, }, ), + ( + 5, + VideoColorFrame { + r: 216, + g: 27, + b: 96, + }, + ), + ( + 6, + VideoColorFrame { + r: 0, + g: 188, + b: 212, + }, + ), + ( + 7, + VideoColorFrame { + r: 205, + g: 220, + b: 57, + }, + ), + ( + 8, + VideoColorFrame { + r: 126, + g: 87, + b: 194, + }, + ), + ( + 9, + VideoColorFrame { + r: 255, + g: 112, + b: 67, + }, + ), + ( + 10, + VideoColorFrame { + r: 38, + g: 166, + b: 154, + }, + ), + ( + 11, + VideoColorFrame { + r: 255, + g: 64, + b: 129, + }, + ), + ( + 12, + VideoColorFrame { + r: 92, + g: 107, + b: 192, + }, + ), + ( + 13, + VideoColorFrame { + r: 255, + g: 235, + b: 59, + }, + ), + ( + 14, + VideoColorFrame { + r: 105, + g: 240, + b: 174, + }, + ), + ( + 15, + VideoColorFrame { + r: 171, + g: 71, + b: 188, + }, + ), + ( + 16, + VideoColorFrame { + r: 3, + g: 169, + b: 244, + }, + ), ] } diff --git a/client/src/sync_probe/analyze/onset_detection/correlation.rs b/client/src/sync_probe/analyze/onset_detection/correlation.rs index f7ae70e..ff884c3 100644 --- a/client/src/sync_probe/analyze/onset_detection/correlation.rs +++ b/client/src/sync_probe/analyze/onset_detection/correlation.rs @@ -286,6 +286,7 @@ pub(crate) fn correlate_coded_segments( &video_indexed, &audio_indexed, &offset_candidates, + event_width_codes, max_pair_gap_s, expected_start_skew_ms, ); @@ -364,6 +365,7 @@ pub(crate) fn correlate_coded_segments( &raw_full_video_indexed, &raw_full_audio_indexed, &candidate_coded_index_offsets(&raw_full_video_indexed, &raw_full_audio_indexed), + event_width_codes, DIAGNOSTIC_CODED_MAX_PAIR_GAP_S, activity_start_delta_ms, ); @@ -468,6 +470,7 @@ fn best_coded_pairs_for_raw_segments( &video_indexed, &audio_indexed, &candidate_coded_index_offsets(&video_indexed, &audio_indexed), + event_width_codes, max_pair_gap_s, expected_start_skew_ms, ); @@ -756,6 +759,8 @@ struct MatchedOnsetPair { audio_time_s: f64, skew_ms: f64, confidence: f64, + server_event_id: Option, + event_code: Option, } impl MatchedOnsetPair { @@ -771,8 +776,16 @@ impl MatchedOnsetPair { audio_time_s, skew_ms, confidence, + server_event_id: None, + event_code: None, } } + + fn with_identity(mut self, event_width_codes: &[u32], event_code: u32) -> Self { + self.server_event_id = unique_event_id_for_code(event_width_codes, event_code); + self.event_code = Some(event_code); + self + } } fn best_pairs_for_index_offsets( @@ -842,6 +855,7 @@ fn best_coded_pairs_for_index_offsets( video_indexed: &BTreeMap, audio_indexed: &BTreeMap, offset_candidates: &[i64], + event_width_codes: &[u32], max_pair_gap_s: f64, expected_start_skew_ms: f64, ) -> Vec { @@ -860,6 +874,7 @@ fn best_coded_pairs_for_index_offsets( .map(|audio| { let skew_ms = (audio.start_s - video.start_s) * 1000.0; MatchedOnsetPair::new(video.start_s, audio.start_s, skew_ms, max_pair_gap_s) + .with_identity(event_width_codes, video.code) }) }) .filter(|pair| pair.skew_ms.abs() <= max_pair_gap_ms) @@ -935,12 +950,10 @@ fn best_coded_pairs_by_time( if let Some((audio_index, audio, skew_ms)) = best_audio { used_audio[audio_index] = true; - pairs.push(MatchedOnsetPair::new( - video.start_s, - audio.start_s, - skew_ms, - max_pair_gap_s, - )); + pairs.push( + MatchedOnsetPair::new(video.start_s, audio.start_s, skew_ms, max_pair_gap_s) + .with_identity(event_width_codes, video_code), + ); } } @@ -951,6 +964,7 @@ fn diagnostic_coded_pairs_for_index_offsets( video_indexed: &BTreeMap, audio_indexed: &BTreeMap, offset_candidates: &[i64], + event_width_codes: &[u32], max_pair_gap_s: f64, expected_start_skew_ms: f64, ) -> Vec { @@ -967,6 +981,7 @@ fn diagnostic_coded_pairs_for_index_offsets( .map(|audio| { let skew_ms = (audio.start_s - video.start_s) * 1000.0; MatchedOnsetPair::new(video.start_s, audio.start_s, skew_ms, max_pair_gap_s) + .with_identity(event_width_codes, video.code) }) }) .filter(|pair| pair.skew_ms.abs() <= max_pair_gap_ms) @@ -1019,7 +1034,9 @@ fn sync_report_from_pairs( .iter() .enumerate() .map(|(event_id, pair)| SyncEventPair { - event_id, + event_id: pair.server_event_id.unwrap_or(event_id), + server_event_id: pair.server_event_id, + event_code: pair.event_code, video_time_s: pair.video_time_s, audio_time_s: pair.audio_time_s, skew_ms: pair.skew_ms, @@ -1062,3 +1079,13 @@ fn sync_report_from_pairs( paired_events, } } + +fn unique_event_id_for_code(event_width_codes: &[u32], event_code: u32) -> Option { + let mut matches = event_width_codes + .iter() + .copied() + .enumerate() + .filter(|(_, code)| *code == event_code); + let (index, _) = matches.next()?; + matches.next().is_none().then_some(index) +} diff --git a/client/src/sync_probe/analyze/onset_detection/tests.rs b/client/src/sync_probe/analyze/onset_detection/tests.rs index d674133..6b95fd4 100644 --- a/client/src/sync_probe/analyze/onset_detection/tests.rs +++ b/client/src/sync_probe/analyze/onset_detection/tests.rs @@ -96,7 +96,7 @@ fn detect_color_coded_video_segments_ignores_generic_bright_changes() { g: 45, b: 45, }, - 40..=45 => VideoColorFrame { + 40..=42 => VideoColorFrame { r: 0, g: 230, b: 118, @@ -224,8 +224,8 @@ fn detect_audio_segments_locks_onto_probe_tone_over_background_hum() { fn detect_coded_audio_segments_uses_probe_tone_frequency_for_event_code() { let mut samples = vec![0i16; 96_000]; add_sine(&mut samples, 48_000, 0.0, 2.0, 120.0, 7_000.0); - add_sine(&mut samples, 48_000, 0.25, 0.07, 660.0, 2_000.0); - add_sine(&mut samples, 48_000, 1.25, 0.07, 1320.0, 2_000.0); + add_sine(&mut samples, 48_000, 0.25, 0.07, 620.0, 2_000.0); + add_sine(&mut samples, 48_000, 1.25, 0.07, 1120.0, 2_000.0); let segments = detect_coded_audio_segments(&samples, 48_000, 10, &[1, 2, 3, 4], 0.12).expect("segments"); diff --git a/client/src/sync_probe/analyze/report.rs b/client/src/sync_probe/analyze/report.rs index 85ae22a..ebe28ec 100644 --- a/client/src/sync_probe/analyze/report.rs +++ b/client/src/sync_probe/analyze/report.rs @@ -40,6 +40,10 @@ pub struct SyncAnalysisReport { #[derive(Clone, Debug, PartialEq, Serialize)] pub struct SyncEventPair { pub event_id: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_event_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub event_code: Option, pub video_time_s: f64, pub audio_time_s: f64, pub skew_ms: f64, diff --git a/common/Cargo.toml b/common/Cargo.toml index fa4990d..9d68469 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.18" +version = "0.19.19" edition = "2024" build = "build.rs" diff --git a/scripts/manual/local_av_stimulus.py b/scripts/manual/local_av_stimulus.py index 3b3e04f..ac22ce9 100755 --- a/scripts/manual/local_av_stimulus.py +++ b/scripts/manual/local_av_stimulus.py @@ -12,7 +12,7 @@ import time import urllib.parse from pathlib import Path -DEFAULT_EVENT_WIDTH_CODES = "1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2" +DEFAULT_EVENT_WIDTH_CODES = "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16" def parse_args() -> argparse.Namespace: @@ -165,13 +165,37 @@ const pulseColors = { 1: '#b81d24', 2: '#007a3d', 3: '#1456b8', - 4: '#b56b00' + 4: '#b56b00', + 5: '#d81b60', + 6: '#00bcd4', + 7: '#cddc39', + 8: '#7e57c2', + 9: '#ff7043', + 10: '#26a69a', + 11: '#ff4081', + 12: '#5c6bc0', + 13: '#ffeb3b', + 14: '#69f0ae', + 15: '#ab47bc', + 16: '#03a9f4' }; const pulseFrequencies = { - 1: 660, - 2: 880, - 3: 1100, - 4: 1320 + 1: 620, + 2: 780, + 3: 940, + 4: 1120, + 5: 1320, + 6: 1540, + 7: 1780, + 8: 2040, + 9: 2320, + 10: 2620, + 11: 2960, + 12: 3340, + 13: 3760, + 14: 4220, + 15: 4740, + 16: 5320 }; function setStatus(message) { statusEl.textContent = message; } @@ -197,7 +221,7 @@ function eventAt(elapsedMs, command) { const offset = sinceWarmup % command.pulse_period_ms; const codes = command.event_width_codes && command.event_width_codes.length ? command.event_width_codes : [1]; const widthCode = codes[pulseIndex % codes.length]; - const width = Math.min(command.pulse_period_ms - 1, command.pulse_width_ms * widthCode); + const width = Math.min(command.pulse_period_ms - 1, command.pulse_width_ms); return { active: offset < width, pulseIndex, widthCode }; } async function runStimulus(command, mode, token) { diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh index 0fb6c59..305c92e 100755 --- a/scripts/manual/run_server_to_rc_mode_matrix.sh +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -83,7 +83,7 @@ LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RM PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20} PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4} -PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2} +PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index df656d0..48b06ba 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -23,7 +23,7 @@ PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20} PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))} PROBE_PULSE_PERIOD_MS=${PROBE_PULSE_PERIOD_MS:-1000} PROBE_PULSE_WIDTH_MS=${PROBE_PULSE_WIDTH_MS:-120} -PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2} +PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0} LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0} # Do not open the UVC host capture far ahead of the probe. The gadget side only @@ -992,7 +992,8 @@ import sys ) = sys.argv[1:] report = json.loads(pathlib.Path(report_path).read_text()) timeline = json.loads(pathlib.Path(timeline_path).read_text()) -server_events = {int(event["event_id"]): event for event in timeline.get("events", [])} +server_event_list = timeline.get("events", []) +server_events = {int(event["event_id"]): event for event in server_event_list} def finite(value): @@ -1018,6 +1019,17 @@ def as_int_or_none(value): return None +server_events_by_code = {} +for event in server_event_list: + code = as_int_or_none(event.get("code")) + if code is None: + continue + server_events_by_code.setdefault(code, []).append(event) +unique_server_events_by_code = { + code: events[0] for code, events in server_events_by_code.items() if len(events) == 1 +} + + def load_json_or_empty(path): try: return json.loads(pathlib.Path(path).read_text()) @@ -1479,12 +1491,75 @@ clock_uncertainty_ms = as_float(clock_alignment.get("uncertainty_ms"), 0.0) audio_delay_ms = as_float(timeline.get("audio_delay_us"), 0.0) / 1000.0 video_delay_ms = as_float(timeline.get("video_delay_us"), 0.0) / 1000.0 + +def corrected_server_capture_time_s(server, key): + server_unix_ns = as_int_or_none(server.get(key)) + if ( + server_unix_ns is None + or theia_to_tethys_offset_ns is None + or capture_start_unix_ns is None + ): + return None + return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0 + + +def nearest_server_event_for_observation(observed, used_event_ids): + observed_video_s = finite(observed.get("video_time_s")) + observed_audio_s = finite(observed.get("audio_time_s")) + if observed_video_s is None and observed_audio_s is None: + return None, None, None + best = None + for server in server_event_list: + event_id = as_int_or_none(server.get("event_id")) + if event_id in used_event_ids: + continue + expected_video_s = corrected_server_capture_time_s(server, "video_feed_unix_ns") + expected_audio_s = corrected_server_capture_time_s(server, "audio_push_unix_ns") + score = 0.0 + parts = 0 + if observed_video_s is not None and expected_video_s is not None: + score += abs(observed_video_s - expected_video_s) + parts += 1 + if observed_audio_s is not None and expected_audio_s is not None: + score += abs(observed_audio_s - expected_audio_s) + parts += 1 + if parts == 0: + continue + normalized_score = score / parts + if best is None or normalized_score < best[0]: + best = (normalized_score, event_id, server) + if best is None: + return None, None, None + return best[2], "nearest_clocked_time", best[0] * 1000.0 + + joined = [] +used_server_event_ids = set() for observed in report.get("paired_events", []): - event_id = int(observed.get("event_id", -1)) - server = server_events.get(event_id) + observed_event_id = as_int_or_none(observed.get("event_id")) + observed_server_event_id = as_int_or_none(observed.get("server_event_id")) + observed_event_code = as_int_or_none(observed.get("event_code")) + server = None + match_method = None + match_score_ms = None + if observed_server_event_id is not None: + server = server_events.get(observed_server_event_id) + match_method = "analyzer_server_event_id" if server else None + if server is None and observed_event_code is not None: + server = unique_server_events_by_code.get(observed_event_code) + match_method = "unique_event_code" if server else None + if server is None: + server, match_method, match_score_ms = nearest_server_event_for_observation( + observed, + used_server_event_ids, + ) + if server is None and observed_event_code is None and observed_event_id is not None: + server = server_events.get(observed_event_id) + match_method = "legacy_pair_event_id" if server else None if not server: continue + event_id = int(server.get("event_id", -1)) + used_server_event_ids.add(event_id) observed_skew_ms = finite(observed.get("skew_ms")) server_feed_delta_ms = finite(server.get("server_feed_delta_ms")) tethys_video_time_s = finite(observed.get("video_time_s")) @@ -1541,6 +1616,11 @@ for observed in report.get("paired_events", []): planned_start_us = int(server.get("planned_start_us", 0)) joined.append({ "event_id": event_id, + "observed_event_id": observed_event_id, + "observed_server_event_id": observed_server_event_id, + "observed_event_code": observed_event_code, + "match_method": match_method, + "match_score_ms": match_score_ms, "code": int(server.get("code", 0)), "event_time_s": planned_start_us / 1_000_000.0, "planned_start_us": planned_start_us, @@ -1759,6 +1839,11 @@ pathlib.Path(output_json_path).write_text(json.dumps(artifact, indent=2, sort_ke with pathlib.Path(output_csv_path).open("w", newline="", encoding="utf-8") as handle: fieldnames = [ "event_id", + "observed_event_id", + "observed_server_event_id", + "observed_event_code", + "match_method", + "match_score_ms", "code", "event_time_s", "tethys_video_time_s", @@ -2653,6 +2738,7 @@ fi if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then remote_fetch_capture="${REMOTE_CAPTURE}" + analysis_status=0 if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then remote_fetch_capture="${REMOTE_CAPTURE%.mkv}-analysis.mkv" echo "==> normalizing remote capture to CFR for analysis" @@ -2690,15 +2776,22 @@ REMOTE_NORMALIZE_SCRIPT echo " ↪ analyzer timeline window: ${analysis_window_arg#--analysis-window-s }" fi echo "==> analyzing capture on ${TETHYS_HOST}" + set +e ssh ${SSH_OPTS} "${TETHYS_HOST}" \ "chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json --event-width-codes '${PROBE_EVENT_WIDTH_CODES}' ${analysis_window_arg}" \ > "${LOCAL_ANALYSIS_JSON}" + analysis_status=$? + set -e fi if [[ "${FETCH_CAPTURE}" != "0" ]]; then echo "==> fetching capture back to ${LOCAL_CAPTURE}" scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}" fi + if [[ "${analysis_status}" -ne 0 ]]; then + echo "remote analysis failed with status ${analysis_status}; capture preserved at ${LOCAL_CAPTURE}" >&2 + exit "${analysis_status}" + fi fi if [[ "${probe_status}" -ne 0 ]]; then @@ -2765,12 +2858,22 @@ pathlib.Path(sys.argv[2]).write_text(summary) with pathlib.Path(sys.argv[3]).open("w", newline="") as handle: writer = csv.DictWriter( handle, - fieldnames=["event_id", "video_time_s", "audio_time_s", "skew_ms", "confidence"], + fieldnames=[ + "event_id", + "server_event_id", + "event_code", + "video_time_s", + "audio_time_s", + "skew_ms", + "confidence", + ], ) writer.writeheader() for event in report.get("paired_events", []): writer.writerow({ "event_id": event.get("event_id"), + "server_event_id": event.get("server_event_id"), + "event_code": event.get("event_code"), "video_time_s": event.get("video_time_s"), "audio_time_s": event.get("audio_time_s"), "skew_ms": event.get("skew_ms"), diff --git a/scripts/manual/run_upstream_mirrored_av_sync.sh b/scripts/manual/run_upstream_mirrored_av_sync.sh index 55d95c9..2ff9f28 100755 --- a/scripts/manual/run_upstream_mirrored_av_sync.sh +++ b/scripts/manual/run_upstream_mirrored_av_sync.sh @@ -22,7 +22,7 @@ PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4} PROBE_PULSE_PERIOD_MS=${PROBE_PULSE_PERIOD_MS:-1000} PROBE_PULSE_WIDTH_MS=${PROBE_PULSE_WIDTH_MS:-120} PROBE_MARKER_TICK_PERIOD=${PROBE_MARKER_TICK_PERIOD:-5} -PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2} +PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} PROBE_AUDIO_GAIN=${PROBE_AUDIO_GAIN:-0.55} LESAVKA_SYNC_CALIBRATION_SEGMENTS_SET=${LESAVKA_SYNC_CALIBRATION_SEGMENTS+x} LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0} diff --git a/server/Cargo.toml b/server/Cargo.toml index 42c7f32..81c694f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.18" +version = "0.19.19" edition = "2024" autobins = false diff --git a/server/src/output_delay_probe.rs b/server/src/output_delay_probe.rs index 64cf571..bf6df17 100644 --- a/server/src/output_delay_probe.rs +++ b/server/src/output_delay_probe.rs @@ -16,9 +16,7 @@ const DEFAULT_DURATION_SECONDS: u32 = 20; const DEFAULT_WARMUP_SECONDS: u32 = 4; const DEFAULT_PULSE_PERIOD_MS: u32 = 1_000; const DEFAULT_PULSE_WIDTH_MS: u32 = 120; -const DEFAULT_EVENT_WIDTH_CODES: &[u32] = &[ - 1, 2, 1, 3, 2, 4, 1, 1, 3, 1, 4, 2, 1, 2, 3, 4, 1, 3, 2, 2, 4, 1, 2, 4, 3, 1, 1, 4, 2, 3, 1, 2, -]; +const DEFAULT_EVENT_WIDTH_CODES: &[u32] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; const AUDIO_SAMPLE_RATE: u32 = 48_000; const AUDIO_CHANNELS: usize = 2; const AUDIO_CHUNK_MS: u64 = 10; @@ -28,7 +26,7 @@ const AUDIO_PILOT_FREQUENCY_HZ: f64 = 180.0; const DARK_FRAME_RGB: Rgb = Rgb { r: 4, g: 8, b: 12 }; const VIDEO_CONTINUITY_BLOCKS: usize = 20; const VIDEO_CONTINUITY_DATA_BITS: usize = 16; -const EVENT_COLORS: [Rgb; 4] = [ +const EVENT_COLORS: [Rgb; 16] = [ Rgb { r: 255, g: 45, @@ -49,8 +47,71 @@ const EVENT_COLORS: [Rgb; 4] = [ g: 179, b: 0, }, + Rgb { + r: 216, + g: 27, + b: 96, + }, + Rgb { + r: 0, + g: 188, + b: 212, + }, + Rgb { + r: 205, + g: 220, + b: 57, + }, + Rgb { + r: 126, + g: 87, + b: 194, + }, + Rgb { + r: 255, + g: 112, + b: 67, + }, + Rgb { + r: 38, + g: 166, + b: 154, + }, + Rgb { + r: 255, + g: 64, + b: 129, + }, + Rgb { + r: 92, + g: 107, + b: 192, + }, + Rgb { + r: 255, + g: 235, + b: 59, + }, + Rgb { + r: 105, + g: 240, + b: 174, + }, + Rgb { + r: 171, + g: 71, + b: 188, + }, + Rgb { + r: 3, + g: 169, + b: 244, + }, +]; +const EVENT_FREQUENCIES_HZ: [f64; 16] = [ + 620.0, 780.0, 940.0, 1_120.0, 1_320.0, 1_540.0, 1_780.0, 2_040.0, 2_320.0, 2_620.0, 2_960.0, + 3_340.0, 3_760.0, 4_220.0, 4_740.0, 5_320.0, ]; -const EVENT_FREQUENCIES_HZ: [f64; 4] = [660.0, 880.0, 1_100.0, 1_320.0]; #[derive(Clone, Copy, Debug)] struct Rgb { @@ -250,8 +311,7 @@ impl ProbeConfig { let period_ns = self.pulse_period.as_nanos().max(1); let pulse_index = (since_warmup.as_nanos() / period_ns) as usize; let pulse_offset_ns = since_warmup.as_nanos() % period_ns; - let code = self.event_width_codes[pulse_index % self.event_width_codes.len()]; - let active_ns = self.pulse_width.as_nanos().saturating_mul(u128::from(code)); + let active_ns = self.pulse_width.as_nanos(); (pulse_offset_ns < active_ns).then(|| self.event_slot_by_id(pulse_index)) } @@ -260,8 +320,7 @@ impl ProbeConfig { let planned_start = self .warmup .saturating_add(duration_mul(self.pulse_period, event_id as u64)); - let planned_end = - planned_start.saturating_add(duration_mul(self.pulse_width, u64::from(code))); + let planned_end = planned_start.saturating_add(self.pulse_width); ProbeEventSlot { event_id, code, @@ -312,8 +371,8 @@ fn parse_event_width_codes(raw: &str) -> Result> { let code = part .parse::() .with_context(|| format!("parsing event width code `{part}`"))?; - if !(1..=4).contains(&code) { - bail!("event width code {code} is unsupported; use values 1..4"); + if !(1..=16).contains(&code) { + bail!("event signature code {code} is unsupported; use values 1..16"); } Ok(code) }) @@ -760,7 +819,7 @@ mod tests { #[test] fn event_codes_reject_unsupported_signatures() { let request = OutputDelayProbeRequest { - event_width_codes: "1,5".to_string(), + event_width_codes: "1,17".to_string(), ..Default::default() };