From db72307edacde6b738a127bd155b861e3914af82 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 4 May 2026 14:50:47 -0300 Subject: [PATCH] test(server-rc): require stronger mode tuning evidence --- Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- .../manual/run_server_to_rc_mode_matrix.sh | 97 +++++++++++++++++-- server/Cargo.toml | 2 +- server/src/output_delay_probe.rs | 34 ++++++- .../client_manual_sync_script_contract.rs | 4 +- 7 files changed, 133 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a71efc..3f26725 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.17" +version = "0.19.18" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.17" +version = "0.19.18" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.17" +version = "0.19.18" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 4cefc9e..2919e7b 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.17" +version = "0.19.18" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 41c24e1..fa4990d 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.17" +version = "0.19.18" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh index c3b9f60..0fb6c59 100755 --- a/scripts/manual/run_server_to_rc_mode_matrix.sh +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -60,8 +60,10 @@ LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1} LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1} LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1} LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1} -LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-3} +LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-8} +LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000} LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80} +LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000} LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000} LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} @@ -186,7 +188,9 @@ run_mode_probe() { LESAVKA_OUTPUT_DELAY_SAVE=0 \ LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \ LESAVKA_OUTPUT_DELAY_MIN_PAIRS="${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \ + LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS="${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \ LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ + LESAVKA_OUTPUT_DELAY_MAX_STEP_US="${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \ LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \ LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \ LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \ @@ -206,7 +210,9 @@ write_tune_candidate_env() { "${mode_result}" \ "${output_env}" \ "${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \ + "${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \ "${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ + "${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \ "${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" import json import math @@ -214,7 +220,15 @@ import pathlib import shlex import sys -result_path, output_env_path, min_pairs_raw, max_drift_raw, min_change_raw = sys.argv[1:6] +( + result_path, + output_env_path, + min_pairs_raw, + max_abs_skew_raw, + max_drift_raw, + max_step_raw, + min_change_raw, +) = sys.argv[1:8] def env_line(key, value): @@ -239,8 +253,10 @@ def as_float(value, default=0.0): result = json.loads(pathlib.Path(result_path).read_text()) calibration = result.get("output_delay_calibration") or {} sync = result.get("sync") or {} -min_pairs = max(1, as_int(min_pairs_raw, 3)) +min_pairs = max(1, as_int(min_pairs_raw, 8)) +max_abs_skew_ms = max(1.0, as_float(max_abs_skew_raw, 1000.0)) max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0)) +max_step_us = max(1, as_int(max_step_raw, 500_000)) min_change_us = max(0, as_int(min_change_raw, 5_000)) current_audio = as_int(result.get("audio_delay_us"), 0) current_video = as_int(result.get("video_delay_us"), 0) @@ -248,16 +264,30 @@ target_audio = as_int(calibration.get("audio_target_offset_us"), current_audio) target_video = as_int(calibration.get("video_target_offset_us"), current_video) paired = as_int(calibration.get("paired_event_count"), as_int(sync.get("paired_event_count"), 0)) drift_ms = as_float(calibration.get("drift_ms"), as_float(sync.get("drift_ms"), 0.0)) +max_abs_skew_ms_observed = as_float( + calibration.get("max_abs_skew_ms"), + as_float(sync.get("p95_abs_skew_ms"), 0.0), +) delta_audio = target_audio - current_audio delta_video = target_video - current_video +raw_delta_us = as_int( + calibration.get("raw_device_delta_us"), + delta_video if abs(delta_video) >= abs(delta_audio) else -delta_audio, +) reasons = [] if result.get("run_status") != 0: reasons.append(f"seed probe exited {result.get('run_status')}") if paired < min_pairs: reasons.append(f"paired_event_count {paired} < {min_pairs}") +if max_abs_skew_ms_observed > max_abs_skew_ms: + reasons.append( + f"max_abs_skew_ms {max_abs_skew_ms_observed:.1f} > {max_abs_skew_ms:.1f}" + ) if abs(drift_ms) > max_drift_ms: reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}") +if abs(raw_delta_us) > max_step_us: + reasons.append(f"raw delay correction {raw_delta_us:+d}us exceeds {max_step_us}us") if target_audio < 0 or target_video < 0: reasons.append( f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}" @@ -277,6 +307,8 @@ values = { "tune_video_delta_us": delta_video, "tune_paired_event_count": paired, "tune_drift_ms": f"{drift_ms:.3f}", + "tune_max_abs_skew_ms": f"{max_abs_skew_ms_observed:.3f}", + "tune_raw_delta_us": raw_delta_us, } with pathlib.Path(output_env_path).open("w") as handle: for key, value in values.items(): @@ -897,6 +929,35 @@ missing = as_int(video.get("estimated_missing_frames"), 0) undecodable = as_int(video.get("undecodable_frames"), 0) duplicates = as_int(video.get("duplicate_frames"), 0) low_rms = as_int(audio_rms.get("low_rms_window_count"), 0) +pair_confidences = [] +for event in report.get("paired_events") or []: + if not isinstance(event, dict): + continue + try: + confidence = float(event.get("confidence")) + except Exception: + continue + if math.isfinite(confidence): + pair_confidences.append(confidence) + + +def median(values, default=0.0): + if not values: + return default + ordered = sorted(values) + mid = len(ordered) // 2 + if len(ordered) % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2.0 + + +activity_start_delta_ms = as_float(report.get("activity_start_delta_ms"), 0.0) +first_skew_ms = as_float(report.get("first_skew_ms"), 0.0) +activity_pair_disagreement_ms = ( + activity_start_delta_ms - first_skew_ms + if as_int(report.get("paired_event_count"), 0) > 0 + else 0.0 +) reasons = [] if run_status != 0: @@ -945,7 +1006,14 @@ artifact = { "p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms"), 0.0), "median_skew_ms": as_float(report.get("median_skew_ms"), 0.0), "drift_ms": as_float(report.get("drift_ms"), 0.0), + "activity_start_delta_ms": activity_start_delta_ms, + "first_skew_ms": first_skew_ms, + "activity_pair_disagreement_ms": activity_pair_disagreement_ms, + "video_event_count": as_int(report.get("video_event_count"), 0), + "audio_event_count": as_int(report.get("audio_event_count"), 0), "paired_event_count": as_int(report.get("paired_event_count"), 0), + "paired_confidence_min": min(pair_confidences) if pair_confidences else 0.0, + "paired_confidence_median": median(pair_confidences), }, "freshness": { "status": freshness_status, @@ -977,6 +1045,7 @@ artifact = { "decision": calibration.get("decision", "unknown"), "target": calibration.get("target", ""), "paired_event_count": as_int(calibration.get("paired_event_count"), 0), + "min_pairs": as_int(calibration.get("min_pairs"), 0), "measured_device_skew_ms": as_float(calibration.get("measured_device_skew_ms"), 0.0), "p95_abs_skew_ms": as_float(calibration.get("p95_abs_skew_ms"), 0.0), "max_abs_skew_ms": as_float(calibration.get("max_abs_skew_ms"), 0.0), @@ -987,6 +1056,9 @@ artifact = { "video_offset_adjust_us": as_int(calibration.get("video_offset_adjust_us"), 0), "audio_target_offset_us": as_int(calibration.get("audio_target_offset_us"), 0), "video_target_offset_us": as_int(calibration.get("video_target_offset_us"), 0), + "raw_device_delta_us": as_int(calibration.get("raw_device_delta_us"), 0), + "bounded_device_delta_us": as_int(calibration.get("bounded_device_delta_us"), 0), + "max_step_us": as_int(calibration.get("max_step_us"), 0), "note": calibration.get("note", ""), }, } @@ -1037,14 +1109,18 @@ for result in results: continue sync = result.get("sync") or {} calibration = result.get("output_delay_calibration") or {} - confirmed = sync.get("passed") is True + required_pairs = calibration.get("min_pairs") or 8 + confirmed = ( + sync.get("passed") is True + and (sync.get("paired_event_count") or 0) >= required_pairs + ) candidate_video = calibration.get("video_target_offset_us") candidate_audio = calibration.get("audio_target_offset_us") video_delay = result.get("video_delay_us") audio_delay = result.get("audio_delay_us") status = "confirmed" if confirmed else "tested" candidate_available = ( - calibration.get("decision") in {"ready", "refused"} + calibration.get("ready") is True and (calibration.get("paired_event_count") or 0) > 0 and isinstance(candidate_video, int) and isinstance(candidate_audio, int) @@ -1069,6 +1145,8 @@ for result in results: "sync_status": sync.get("status"), "median_skew_ms": sync.get("median_skew_ms"), "paired_event_count": sync.get("paired_event_count"), + "paired_confidence_median": sync.get("paired_confidence_median"), + "activity_pair_disagreement_ms": sync.get("activity_pair_disagreement_ms"), } summary = { @@ -1195,6 +1273,13 @@ for result in results: f"video_target={calibration.get('video_target_offset_us', 0)}us " f"audio_target={calibration.get('audio_target_offset_us', 0)}us" ) + lines.append( + " sync evidence: " + f"video_onsets={sync.get('video_event_count', 0)} audio_onsets={sync.get('audio_event_count', 0)} " + f"pairs={sync.get('paired_event_count', 0)} " + f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} " + f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms" + ) if "seed_video_delay_us" in result or "seed_audio_delay_us" in result: lines.append( " tuning: " @@ -1223,7 +1308,7 @@ echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}" echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}" -echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" +echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}" echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s" diff --git a/server/Cargo.toml b/server/Cargo.toml index 43214cd..42c7f32 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.17" +version = "0.19.18" edition = "2024" autobins = false diff --git a/server/src/output_delay_probe.rs b/server/src/output_delay_probe.rs index 51fa4dc..64cf571 100644 --- a/server/src/output_delay_probe.rs +++ b/server/src/output_delay_probe.rs @@ -738,7 +738,8 @@ fn duration_mul(duration: Duration, count: u64) -> Duration { #[cfg(test)] mod tests { use super::{ - OutputDelayProbeTimeline, ProbeConfig, duration_us, render_audio_chunk, unix_ns_from_start, + DARK_FRAME_RGB, OutputDelayProbeTimeline, ProbeConfig, duration_us, probe_color_for_code, + render_audio_chunk, unix_ns_from_start, }; use crate::camera::{CameraCodec, CameraConfig, CameraOutput}; use lesavka_common::lesavka::OutputDelayProbeRequest; @@ -787,6 +788,37 @@ mod tests { assert!(rms_i16_le(&active) > rms_i16_le(&idle) * 10.0); } + #[test] + fn generated_video_and_audio_share_the_same_event_schedule() { + let config = ProbeConfig::from_request(&OutputDelayProbeRequest { + duration_seconds: 6, + warmup_seconds: 1, + pulse_period_ms: 1_000, + pulse_width_ms: 120, + event_width_codes: "2".to_string(), + audio_delay_us: 0, + video_delay_us: 0, + }) + .expect("config"); + let idle_pts = Duration::from_millis(500); + let active_pts = Duration::from_secs(1); + + let idle_color = probe_color_for_code(config.event_code_at(idle_pts)); + let active_color = probe_color_for_code(config.event_code_at(active_pts)); + let idle_audio = render_audio_chunk(&config, idle_pts, 480); + let active_audio = render_audio_chunk(&config, active_pts, 480); + + assert_eq!( + (idle_color.r, idle_color.g, idle_color.b), + (DARK_FRAME_RGB.r, DARK_FRAME_RGB.g, DARK_FRAME_RGB.b) + ); + assert_ne!( + (active_color.r, active_color.g, active_color.b), + (DARK_FRAME_RGB.r, DARK_FRAME_RGB.g, DARK_FRAME_RGB.b) + ); + assert!(rms_i16_le(&active_audio) > rms_i16_le(&idle_audio) * 10.0); + } + #[test] fn timeline_exports_wall_clock_fields_for_freshness() { let config = ProbeConfig::from_request(&OutputDelayProbeRequest { diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index b20d0c0..67d07cc 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -239,7 +239,9 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}", "LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}", "LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}", - "LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-3}", + "LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-8}", + "LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}", + "LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}", "LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}", "Theia sudo password for %s", "==> priming remote sudo on ${LESAVKA_SERVER_HOST}",