From 40287aca33a51f9229601e3e8247a40c434798ed Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Wed, 6 May 2026 00:29:00 -0300 Subject: [PATCH] calibration: aggregate repeated RC mode probes --- Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- .../manual/run_server_to_rc_mode_matrix.sh | 826 +++++++++++++----- server/Cargo.toml | 2 +- .../client_manual_sync_script_contract.rs | 45 +- 6 files changed, 652 insertions(+), 231 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f69ceb2..e42784b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.27" +version = "0.19.28" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.27" +version = "0.19.28" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.27" +version = "0.19.28" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 56e1944..cae3796 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.27" +version = "0.19.28" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 6228cf6..9c46872 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.27" +version = "0.19.28" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh index fa3c31c..de256cc 100755 --- a/scripts/manual/run_server_to_rc_mode_matrix.sh +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -66,6 +66,14 @@ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS: LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80} LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000} LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000} +LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1} +LESAVKA_SERVER_RC_VERBOSE_PROBES=${LESAVKA_SERVER_RC_VERBOSE_PROBES:-1} +LESAVKA_SERVER_RC_STATIC_MIN_RUNS=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS:-3} +LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US:-30000} +LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS:-35} +LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS:-20} +LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS:-1} +LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS:-0} LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100} @@ -117,7 +125,13 @@ MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv" MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt" MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json" MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env" +MATRIX_STATIC_JSON="${MATRIX_REPORT_DIR}/mode-static-calibration.json" +MATRIX_STATIC_CSV="${MATRIX_REPORT_DIR}/mode-static-calibration.csv" +MATRIX_STATIC_TXT="${MATRIX_REPORT_DIR}/mode-static-calibration.txt" +MATRIX_STATIC_ENV="${MATRIX_REPORT_DIR}/mode-static-calibration.env" +MATRIX_RUN_LOG="${MATRIX_REPORT_DIR}/mode-matrix-run.log" mkdir -p "${MATRIX_REPORT_DIR}" +exec > >(tee -a "${MATRIX_RUN_LOG}") 2>&1 mode_id() { local mode=$1 @@ -194,52 +208,60 @@ run_mode_probe() { fi fi + local -a probe_env=( + "TETHYS_HOST=${TETHYS_HOST}" + "LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST}" + "LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST}" + "LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR}" + "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME}" + "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN}" + "SSH_OPTS=${SSH_OPTS}" + "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}" + "REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}" + "REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}" + "REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE}" + "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" + "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" + "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}" + "PROBE_PREBUILD=0" + "VIDEO_SIZE=${width}x${height}" + "VIDEO_FPS=${fps}" + "VIDEO_FORMAT=mjpeg" + "REMOTE_EXPECT_UVC_WIDTH=${width}" + "REMOTE_EXPECT_UVC_HEIGHT=${height}" + "REMOTE_EXPECT_UVC_FPS=${fps}" + "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}" + "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}" + "LESAVKA_OUTPUT_DELAY_APPLY=0" + "LESAVKA_OUTPUT_DELAY_SAVE=0" + "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0" + "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${min_pairs}" + "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" + "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" + "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" + "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" + "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" + "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" + "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}" + "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES}" + "PROBE_CONDITIONING_SECONDS=${conditioning_seconds}" + "PROBE_CONDITIONING_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}" + "PROBE_CONDITIONING_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}" + "PROBE_CONDITIONING_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}" + "PROBE_DURATION_SECONDS=${probe_duration_seconds}" + "PROBE_WARMUP_SECONDS=${probe_warmup_seconds}" + "ANALYSIS_TIMELINE_WINDOW=${analysis_timeline_window}" + "LOCAL_OUTPUT_DIR=${output_dir}" + ) + set +e - TETHYS_HOST="${TETHYS_HOST}" \ - LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \ - LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \ - LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \ - LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \ - LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \ - SSH_OPTS="${SSH_OPTS}" \ - REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \ - REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \ - REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \ - REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \ - REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ - REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \ - REMOTE_CAPTURE_READY_SETTLE_SECONDS="${REMOTE_CAPTURE_READY_SETTLE_SECONDS}" \ - PROBE_PREBUILD=0 \ - VIDEO_SIZE="${width}x${height}" \ - VIDEO_FPS="${fps}" \ - VIDEO_FORMAT=mjpeg \ - REMOTE_EXPECT_UVC_WIDTH="${width}" \ - REMOTE_EXPECT_UVC_HEIGHT="${height}" \ - REMOTE_EXPECT_UVC_FPS="${fps}" \ - LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${audio_delay_us}" \ - LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \ - LESAVKA_OUTPUT_DELAY_APPLY=0 \ - LESAVKA_OUTPUT_DELAY_SAVE=0 \ - LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \ - LESAVKA_OUTPUT_DELAY_MIN_PAIRS="${min_pairs}" \ - LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS="${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \ - LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ - LESAVKA_OUTPUT_DELAY_MAX_STEP_US="${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \ - LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \ - LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \ - LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \ - LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS="${min_pairs}" \ - PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \ - PROBE_CONDITIONING_SECONDS="${conditioning_seconds}" \ - PROBE_CONDITIONING_WARMUP_SECONDS="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}" \ - PROBE_CONDITIONING_GAP_SECONDS="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}" \ - PROBE_CONDITIONING_EVENT_WIDTH_CODES="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}" \ - PROBE_DURATION_SECONDS="${probe_duration_seconds}" \ - PROBE_WARMUP_SECONDS="${probe_warmup_seconds}" \ - ANALYSIS_TIMELINE_WINDOW="${analysis_timeline_window}" \ - LOCAL_OUTPUT_DIR="${output_dir}" \ - "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}" - RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]} + if [[ "${LESAVKA_SERVER_RC_VERBOSE_PROBES}" == "0" ]]; then + env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" >"${log_path}" 2>&1 + RUN_MODE_PROBE_STATUS=$? + else + env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}" + RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]} + fi set -e } @@ -1552,8 +1574,63 @@ pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=Tr PY } +annotate_mode_result() { + local output_json=$1 + local mode_id=$2 + local mode_run_index=$3 + local repeat_index=$4 + local repeat_count=$5 + local matrix_sequence=$6 + local phase=$7 + + python3 - <<'PY' \ + "${output_json}" \ + "${mode_id}" \ + "${mode_run_index}" \ + "${repeat_index}" \ + "${repeat_count}" \ + "${matrix_sequence}" \ + "${phase}" +import json +import pathlib +import sys + +( + output_json_raw, + mode_id, + mode_run_index_raw, + repeat_index_raw, + repeat_count_raw, + matrix_sequence_raw, + phase, +) = sys.argv[1:] + + +def as_int(value, default=0): + try: + return int(str(value).strip()) + except Exception: + return default + + +path = pathlib.Path(output_json_raw) +result = json.loads(path.read_text()) +result.update( + { + "mode_id": mode_id, + "mode_run_index": as_int(mode_run_index_raw), + "repeat_index": as_int(repeat_index_raw), + "repeat_count": as_int(repeat_count_raw), + "matrix_sequence": as_int(matrix_sequence_raw), + "probe_phase": phase, + } +) +path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") +PY +} + summarize_matrix() { - python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" + python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" "${MATRIX_STATIC_JSON}" "${MATRIX_STATIC_CSV}" "${MATRIX_STATIC_TXT}" "${MATRIX_STATIC_ENV}" "${LESAVKA_SERVER_RC_STATIC_MIN_RUNS}" "${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US}" "${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS}" import csv import json import pathlib @@ -1566,6 +1643,49 @@ summary_csv = pathlib.Path(sys.argv[3]) summary_txt = pathlib.Path(sys.argv[4]) delay_json = pathlib.Path(sys.argv[5]) delay_env = pathlib.Path(sys.argv[6]) +static_json = pathlib.Path(sys.argv[7]) +static_csv = pathlib.Path(sys.argv[8]) +static_txt = pathlib.Path(sys.argv[9]) +static_env = pathlib.Path(sys.argv[10]) +static_min_runs = int(sys.argv[11]) +static_max_spread_us = int(sys.argv[12]) +static_max_p95_skew_ms = float(sys.argv[13]) +static_max_median_skew_ms = float(sys.argv[14]) +static_require_freshness = sys.argv[15].strip().lower() not in {"", "0", "false", "no", "off"} +static_require_smoothness = sys.argv[16].strip().lower() not in {"", "0", "false", "no", "off"} + + +def as_int(value, default=0): + try: + return int(str(value).strip()) + except Exception: + return default + + +def as_float(value, default=0.0): + try: + return float(str(value).strip()) + except Exception: + return default + + +def median(values, default=None): + cleaned = sorted(value for value in values if value is not None) + if not cleaned: + return default + mid = len(cleaned) // 2 + if len(cleaned) % 2: + return cleaned[mid] + return (cleaned[mid - 1] + cleaned[mid]) / 2.0 + + +def median_int(values, default=None): + value = median(values, default) + if value is None: + return None + return int(round(value)) + + results = [] for path in sorted(root.glob("*/mode-result.json")): try: @@ -1582,57 +1702,192 @@ for path in sorted(root.glob("*/mode-result.json")): pass if tuned_path.exists(): result["tuned_result_json"] = str(tuned_path) + result["result_json"] = str(path) results.append(result) except Exception: continue +results.sort( + key=lambda item: ( + item.get("mode") or "", + as_int(item.get("mode_run_index"), as_int(item.get("matrix_sequence"), 0)), + as_int(item.get("matrix_sequence"), 0), + ) +) -delay_recommendations = {} -video_delay_entries = [] -audio_delay_entries = [] +mode_groups = {} for result in results: mode = result.get("mode") if not mode: continue + mode_groups.setdefault(mode, []).append(result) + + +def static_eligibility(result): sync = result.get("sync") or {} + freshness = result.get("freshness") or {} calibration = result.get("output_delay_calibration") or {} - required_pairs = calibration.get("min_pairs") or 13 - confirmed = ( - sync.get("passed") is True - and (sync.get("paired_event_count") or 0) >= required_pairs - ) - candidate_video = calibration.get("video_target_offset_us") - candidate_audio = calibration.get("audio_target_offset_us") - video_delay = result.get("video_delay_us") - audio_delay = result.get("audio_delay_us") - status = "confirmed" if confirmed else "tested" - candidate_available = ( - calibration.get("ready") is True - and (calibration.get("paired_event_count") or 0) > 0 - and isinstance(candidate_video, int) - and isinstance(candidate_audio, int) - ) - if not confirmed and candidate_available: - video_delay = candidate_video - audio_delay = candidate_audio - status = "candidate_unconfirmed" - elif not confirmed: - status = "unavailable" - video_delay = None - audio_delay = None - if status != "unavailable" and isinstance(video_delay, int) and isinstance(audio_delay, int): - video_delay_entries.append(f"{mode}={video_delay}") - audio_delay_entries.append(f"{mode}={audio_delay}") - delay_recommendations[mode] = { + reasons = [] + required_pairs = as_int(calibration.get("min_pairs"), 13) or 13 + if result.get("passed") is not True: + reasons.append("probe did not pass all required gates") + if sync.get("passed") is not True: + reasons.append(f"sync did not pass ({sync.get('status', 'unknown')})") + if as_int(sync.get("paired_event_count"), 0) < required_pairs: + reasons.append(f"paired events {sync.get('paired_event_count', 0)} < {required_pairs}") + if as_float(sync.get("p95_abs_skew_ms"), 0.0) > static_max_p95_skew_ms: + reasons.append( + f"p95 skew {as_float(sync.get('p95_abs_skew_ms'), 0.0):.1f}ms > {static_max_p95_skew_ms:.1f}ms" + ) + if abs(as_float(sync.get("median_skew_ms"), 0.0)) > static_max_median_skew_ms: + reasons.append( + f"median skew {as_float(sync.get('median_skew_ms'), 0.0):+.1f}ms outside +/-{static_max_median_skew_ms:.1f}ms" + ) + if static_require_freshness and freshness.get("status") != "pass": + reasons.append(f"freshness did not pass ({freshness.get('status', 'unknown')})") + if static_require_smoothness and result.get("smoothness_warnings"): + reasons.append("smoothness warnings present") + if not isinstance(result.get("video_delay_us"), int) or not isinstance(result.get("audio_delay_us"), int): + reasons.append("tested delay values unavailable") + return reasons + + +static_modes = {} +delay_recommendations = {} +video_delay_entries = [] +audio_delay_entries = [] +static_video_entries = [] +static_audio_entries = [] +for mode, mode_results in sorted(mode_groups.items()): + eligible = [] + rejected = [] + for result in mode_results: + reasons = static_eligibility(result) + if reasons: + rejected.append( + { + "mode_run_index": result.get("mode_run_index"), + "artifact_dir": result.get("artifact_dir"), + "reasons": reasons, + "tested_video_delay_us": result.get("video_delay_us"), + "tested_audio_delay_us": result.get("audio_delay_us"), + "p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"), + "median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"), + } + ) + else: + eligible.append(result) + + tested_video = [result.get("video_delay_us") for result in eligible] + tested_audio = [result.get("audio_delay_us") for result in eligible] + target_video = [ + (result.get("output_delay_calibration") or {}).get("video_target_offset_us") + for result in eligible + if isinstance((result.get("output_delay_calibration") or {}).get("video_target_offset_us"), int) + ] + target_audio = [ + (result.get("output_delay_calibration") or {}).get("audio_target_offset_us") + for result in eligible + if isinstance((result.get("output_delay_calibration") or {}).get("audio_target_offset_us"), int) + ] + tested_video_spread = (max(tested_video) - min(tested_video)) if tested_video else None + tested_audio_spread = (max(tested_audio) - min(tested_audio)) if tested_audio else None + target_video_spread = (max(target_video) - min(target_video)) if target_video else None + recommended_video = median_int(tested_video) + recommended_audio = median_int(tested_audio) + status_reasons = [] + if len(eligible) < static_min_runs: + status_reasons.append(f"eligible runs {len(eligible)} < {static_min_runs}") + if tested_video_spread is not None and tested_video_spread > static_max_spread_us: + status_reasons.append(f"tested video delay spread {tested_video_spread}us > {static_max_spread_us}us") + if tested_audio_spread is not None and tested_audio_spread > static_max_spread_us: + status_reasons.append(f"tested audio delay spread {tested_audio_spread}us > {static_max_spread_us}us") + if recommended_video is None or recommended_audio is None: + status_reasons.append("no recommended delay could be derived") + if len(eligible) < static_min_runs: + status = "needs_more_runs" + elif status_reasons: + status = "unstable" + else: + status = "ready" + + p95_values = [as_float((result.get("sync") or {}).get("p95_abs_skew_ms"), 0.0) for result in eligible] + median_skews = [as_float((result.get("sync") or {}).get("median_skew_ms"), 0.0) for result in eligible] + freshness_budgets = [ + as_float((result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"), 0.0) + for result in eligible + ] + static_entry = { "status": status, - "audio_delay_us": audio_delay, - "video_delay_us": video_delay, - "tested_audio_delay_us": result.get("audio_delay_us"), - "tested_video_delay_us": result.get("video_delay_us"), - "sync_status": sync.get("status"), - "median_skew_ms": sync.get("median_skew_ms"), - "paired_event_count": sync.get("paired_event_count"), - "paired_confidence_median": sync.get("paired_confidence_median"), - "activity_pair_disagreement_ms": sync.get("activity_pair_disagreement_ms"), + "ready": status == "ready", + "reasons": status_reasons, + "mode": mode, + "total_runs": len(mode_results), + "eligible_runs": len(eligible), + "rejected_runs": rejected, + "recommended_video_delay_us": recommended_video, + "recommended_audio_delay_us": recommended_audio, + "tested_video_delay_us": tested_video, + "tested_audio_delay_us": tested_audio, + "tested_video_delay_min_us": min(tested_video) if tested_video else None, + "tested_video_delay_max_us": max(tested_video) if tested_video else None, + "tested_video_delay_spread_us": tested_video_spread, + "tested_audio_delay_spread_us": tested_audio_spread, + "target_video_delay_us": target_video, + "target_audio_delay_us": target_audio, + "target_video_delay_median_us": median_int(target_video), + "target_audio_delay_median_us": median_int(target_audio), + "target_video_delay_spread_us": target_video_spread, + "sync_p95_abs_skew_ms_max": max(p95_values) if p95_values else None, + "sync_p95_abs_skew_ms_median": median(p95_values), + "sync_abs_median_skew_ms_max": max((abs(value) for value in median_skews), default=None), + "freshness_budget_ms_max": max(freshness_budgets) if freshness_budgets else None, + "eligible_artifact_dirs": [result.get("artifact_dir") for result in eligible], + "eligible_result_json": [result.get("result_json") for result in eligible], + } + static_modes[mode] = static_entry + + if status == "ready" and isinstance(recommended_video, int) and isinstance(recommended_audio, int): + static_video_entries.append(f"{mode}={recommended_video}") + static_audio_entries.append(f"{mode}={recommended_audio}") + + delay_video = recommended_video + delay_audio = recommended_audio + recommendation_status = "static_ready" if status == "ready" else status + if status != "ready": + fallback = (eligible or mode_results)[-1] + fallback_calibration = fallback.get("output_delay_calibration") or {} + fallback_video_target = fallback_calibration.get("video_target_offset_us") + fallback_audio_target = fallback_calibration.get("audio_target_offset_us") + if not isinstance(delay_video, int) or not isinstance(delay_audio, int): + if ( + fallback_calibration.get("ready") is True + and isinstance(fallback_video_target, int) + and isinstance(fallback_audio_target, int) + ): + delay_video = fallback_video_target + delay_audio = fallback_audio_target + recommendation_status = "candidate_unconfirmed" + else: + delay_video = fallback.get("video_delay_us") + delay_audio = fallback.get("audio_delay_us") + recommendation_status = "tested_unstable" + else: + recommendation_status = f"candidate_{status}" + if isinstance(delay_video, int) and isinstance(delay_audio, int): + video_delay_entries.append(f"{mode}={delay_video}") + audio_delay_entries.append(f"{mode}={delay_audio}") + delay_recommendations[mode] = { + "status": recommendation_status, + "static_status": status, + "audio_delay_us": delay_audio, + "video_delay_us": delay_video, + "eligible_runs": len(eligible), + "total_runs": len(mode_results), + "tested_video_delay_spread_us": tested_video_spread, + "sync_p95_abs_skew_ms_max": static_entry["sync_p95_abs_skew_ms_max"], + "sync_abs_median_skew_ms_max": static_entry["sync_abs_median_skew_ms_max"], + "freshness_budget_ms_max": static_entry["freshness_budget_ms_max"], + "reasons": status_reasons, } summary = { @@ -1642,6 +1897,12 @@ summary = { "mode_count": len(results), "delay_recommendations_json": str(delay_json), "delay_recommendations_env": str(delay_env), + "static_calibration_json": str(static_json), + "static_calibration_csv": str(static_csv), + "static_calibration_txt": str(static_txt), + "static_calibration_env": str(static_env), + "static_ready": bool(static_modes) and all(entry.get("ready") for entry in static_modes.values()), + "static_calibration": static_modes, "delay_recommendations": delay_recommendations, "results": results, } @@ -1666,6 +1927,37 @@ delay_json.write_text( ) + "\n" ) +static_json.write_text( + json.dumps( + { + "schema": "lesavka.server-rc-static-calibration.v1", + "artifact_dir": str(root), + "ready": summary["static_ready"], + "criteria": { + "min_runs": static_min_runs, + "max_spread_us": static_max_spread_us, + "max_p95_skew_ms": static_max_p95_skew_ms, + "max_median_skew_ms": static_max_median_skew_ms, + "require_freshness": static_require_freshness, + "require_smoothness": static_require_smoothness, + }, + "video_delays_us": { + mode: entry.get("recommended_video_delay_us") + for mode, entry in static_modes.items() + if entry.get("ready") + }, + "audio_delays_us": { + mode: entry.get("recommended_audio_delay_us") + for mode, entry in static_modes.items() + if entry.get("ready") + }, + "modes": static_modes, + }, + indent=2, + sort_keys=True, + ) + + "\n" +) delay_env.write_text( "LESAVKA_SERVER_RC_MODE_DELAYS_US=" + shlex.quote(",".join(video_delay_entries)) @@ -1674,9 +1966,20 @@ delay_env.write_text( + shlex.quote(",".join(audio_delay_entries)) + "\n" ) +static_env.write_text( + "LESAVKA_SERVER_RC_MODE_DELAYS_US=" + + shlex.quote(",".join(static_video_entries)) + + "\n" + + "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=" + + shlex.quote(",".join(static_audio_entries)) + + "\n" +) fieldnames = [ "mode", + "mode_run_index", + "repeat_index", + "matrix_sequence", "passed", "seed_video_delay_us", "seed_audio_delay_us", @@ -1702,6 +2005,7 @@ fieldnames = [ "calibration_video_target_offset_us", "calibration_audio_target_offset_us", "artifact_dir", + "result_json", ] with summary_csv.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) @@ -1709,6 +2013,9 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle: for result in results: writer.writerow({ "mode": result.get("mode"), + "mode_run_index": result.get("mode_run_index"), + "repeat_index": result.get("repeat_index"), + "matrix_sequence": result.get("matrix_sequence"), "passed": result.get("passed"), "seed_video_delay_us": result.get("seed_video_delay_us"), "seed_audio_delay_us": result.get("seed_audio_delay_us"), @@ -1734,12 +2041,60 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle: "calibration_video_target_offset_us": (result.get("output_delay_calibration") or {}).get("video_target_offset_us"), "calibration_audio_target_offset_us": (result.get("output_delay_calibration") or {}).get("audio_target_offset_us"), "artifact_dir": result.get("artifact_dir"), + "result_json": result.get("result_json"), }) +static_fieldnames = [ + "mode", + "status", + "ready", + "eligible_runs", + "total_runs", + "recommended_video_delay_us", + "recommended_audio_delay_us", + "tested_video_delay_min_us", + "tested_video_delay_max_us", + "tested_video_delay_spread_us", + "target_video_delay_median_us", + "target_video_delay_spread_us", + "sync_p95_abs_skew_ms_max", + "sync_p95_abs_skew_ms_median", + "sync_abs_median_skew_ms_max", + "freshness_budget_ms_max", + "reasons", +] +with static_csv.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=static_fieldnames) + writer.writeheader() + for mode, entry in sorted(static_modes.items()): + writer.writerow( + { + "mode": mode, + "status": entry.get("status"), + "ready": entry.get("ready"), + "eligible_runs": entry.get("eligible_runs"), + "total_runs": entry.get("total_runs"), + "recommended_video_delay_us": entry.get("recommended_video_delay_us"), + "recommended_audio_delay_us": entry.get("recommended_audio_delay_us"), + "tested_video_delay_min_us": entry.get("tested_video_delay_min_us"), + "tested_video_delay_max_us": entry.get("tested_video_delay_max_us"), + "tested_video_delay_spread_us": entry.get("tested_video_delay_spread_us"), + "target_video_delay_median_us": entry.get("target_video_delay_median_us"), + "target_video_delay_spread_us": entry.get("target_video_delay_spread_us"), + "sync_p95_abs_skew_ms_max": entry.get("sync_p95_abs_skew_ms_max"), + "sync_p95_abs_skew_ms_median": entry.get("sync_p95_abs_skew_ms_median"), + "sync_abs_median_skew_ms_max": entry.get("sync_abs_median_skew_ms_max"), + "freshness_budget_ms_max": entry.get("freshness_budget_ms_max"), + "reasons": "; ".join(entry.get("reasons") or []), + } + ) + lines = [ f"Server-to-RC mode matrix for {root}", - f"- modes: {len(results)}", + f"- runs: {len(results)}", + f"- unique modes: {len(mode_groups)}", f"- verdict: {'pass' if summary['passed'] else 'fail'}", + f"- static calibration: {'ready' if summary['static_ready'] else 'needs-more-data'}", ] for result in results: sync = result.get("sync") or {} @@ -1748,7 +2103,7 @@ for result in results: calibration = result.get("output_delay_calibration") or {} marker = "PASS" if result.get("passed") else "FAIL" lines.append( - f"- {marker} {result.get('mode')}: " + f"- {marker} {result.get('mode')} run={result.get('mode_run_index', '?')}: " f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; " f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; " f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; " @@ -1824,7 +2179,41 @@ for result in results: if video_delay_entries or audio_delay_entries: lines.append(f"- recommended video delays: {','.join(video_delay_entries)}") lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}") +static_lines = [ + f"Server-to-RC static calibration for {root}", + f"- verdict: {'ready' if summary['static_ready'] else 'needs-more-data'}", + ( + "- criteria: " + f"min_runs={static_min_runs} " + f"max_spread_us={static_max_spread_us} " + f"max_p95_skew_ms={static_max_p95_skew_ms:.1f} " + f"max_median_skew_ms={static_max_median_skew_ms:.1f} " + f"require_freshness={static_require_freshness} " + f"require_smoothness={static_require_smoothness}" + ), +] +for mode, entry in sorted(static_modes.items()): + marker = "READY" if entry.get("ready") else "HOLD" + reasons = "; ".join(entry.get("reasons") or []) + if not reasons and entry.get("ready"): + reasons = "all stability criteria met" + static_lines.append( + f"- {marker} {mode}: " + f"runs={entry.get('eligible_runs')}/{entry.get('total_runs')} " + f"video={entry.get('recommended_video_delay_us')}us " + f"audio={entry.get('recommended_audio_delay_us')}us " + f"spread={entry.get('tested_video_delay_spread_us')}us " + f"p95_max={entry.get('sync_p95_abs_skew_ms_max') or 0.0:.1f}ms " + f"median_abs_max={entry.get('sync_abs_median_skew_ms_max') or 0.0:.1f}ms " + f"freshness_budget_max={entry.get('freshness_budget_ms_max') or 0.0:.1f}ms" + ) + static_lines.append(f" reason: {reasons}") +if static_video_entries or static_audio_entries: + static_lines.append(f"- static video delays: {','.join(static_video_entries)}") + static_lines.append(f"- static audio delays: {','.join(static_audio_entries)}") +lines.extend(["", *static_lines]) summary_txt.write_text("\n".join(lines) + "\n") +static_txt.write_text("\n".join(static_lines) + "\n") print("\n".join(lines)) PY } @@ -1838,10 +2227,12 @@ fi echo "==> server-to-RC mode matrix" echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}" echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}" +echo " ↪ repeat_count=${LESAVKA_SERVER_RC_REPEAT_COUNT} verbose_probes=${LESAVKA_SERVER_RC_VERBOSE_PROBES}" echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}" echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" +echo " ↪ static_min_runs=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS} static_max_spread_us=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US} static_max_p95_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS} static_max_median_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}" echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}" echo " ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}" echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} mode=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s" @@ -1850,144 +2241,164 @@ echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SER echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s" echo " ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s" echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}" +echo " ↪ matrix_run_log=${MATRIX_RUN_LOG}" prime_remote_sudo sleep_start_delay prebuild_probe_tools +if ! [[ "${LESAVKA_SERVER_RC_REPEAT_COUNT}" =~ ^[0-9]+$ ]] || (( LESAVKA_SERVER_RC_REPEAT_COUNT < 1 )); then + printf 'LESAVKA_SERVER_RC_REPEAT_COUNT must be a positive integer; got %s\n' "${LESAVKA_SERVER_RC_REPEAT_COUNT}" >&2 + exit 64 +fi + +declare -A mode_run_counts=() +matrix_sequence=0 IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}" for mode in "${modes[@]}"; do mode="${mode//[[:space:]]/}" [[ -n "${mode}" ]] || continue read -r width height fps < <(parse_mode "${mode}") - video_delay_us="$(lookup_video_delay_us "${mode}")" - audio_delay_us="$(lookup_audio_delay_us "${mode}")" - id="$(mode_id "${mode}")" - mode_dir="${MATRIX_REPORT_DIR}/${id}" - mode_log="${mode_dir}/mode-run.log" - mode_result="${mode_dir}/mode-result.json" - seed_result="${mode_dir}/mode-result-seed.json" - readiness_dir="${mode_dir}/signal-readiness" - readiness_log="${readiness_dir}/signal-readiness-run.log" - readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json" - tuned_log="${mode_dir}/mode-tuned-run.log" - tuned_result="${mode_dir}/mode-result-tuned.json" - tune_env="${mode_dir}/mode-tune-candidate.env" - mkdir -p "${mode_dir}" - echo "==> mode ${mode}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}" - reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" - wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}" + for repeat_index in $(seq 1 "${LESAVKA_SERVER_RC_REPEAT_COUNT}"); do + mode_run_counts["${mode}"]=$(( ${mode_run_counts["${mode}"]:-0} + 1 )) + mode_run_index=${mode_run_counts["${mode}"]} + matrix_sequence=$((matrix_sequence + 1)) + video_delay_us="$(lookup_video_delay_us "${mode}")" + audio_delay_us="$(lookup_audio_delay_us "${mode}")" + id_base="$(mode_id "${mode}")" + run_label="$(printf '%02d' "${mode_run_index}")" + id="${id_base}__run${run_label}" + mode_dir="${MATRIX_REPORT_DIR}/${id}" + mode_log="${mode_dir}/mode-run.log" + mode_result="${mode_dir}/mode-result.json" + seed_result="${mode_dir}/mode-result-seed.json" + readiness_dir="${mode_dir}/signal-readiness" + readiness_log="${readiness_dir}/signal-readiness-run.log" + readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json" + tuned_log="${mode_dir}/mode-tuned-run.log" + tuned_result="${mode_dir}/mode-result-tuned.json" + tune_env="${mode_dir}/mode-tune-candidate.env" + mkdir -p "${mode_dir}" - if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" != "separate" ]]; then - echo "==> mode ${mode}: using same-capture signal conditioning before measured probe" - fi + echo "==> mode ${mode} run ${mode_run_index} repeat ${repeat_index}/${LESAVKA_SERVER_RC_REPEAT_COUNT}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}" + reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" + wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}" - if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" == "separate" ]]; then - mkdir -p "${readiness_dir}" - echo "==> mode ${mode}: proving Tethys signal readiness before measured probe" - readiness_pass=1 - readiness_status=0 - readiness_artifact_dir="" - readiness_reason="no readiness attempts ran" - readiness_attempt_jsons=() - for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do - attempt_dir="${readiness_dir}/attempt-${readiness_attempt}" - readiness_log="${attempt_dir}/signal-readiness-run.log" - readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json" - mkdir -p "${attempt_dir}" - echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events" - run_mode_probe \ - "${width}" \ - "${height}" \ - "${fps}" \ - "${audio_delay_us}" \ - "${video_delay_us}" \ - "${attempt_dir}" \ - "${readiness_log}" \ - "${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \ - "${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \ - "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ - 0 - readiness_status=${RUN_MODE_PROBE_STATUS} - readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")" - set +e - readiness_reason="$( - write_signal_readiness_attempt_result \ - "${readiness_attempt}" \ - "${readiness_artifact_dir}" \ + if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" != "separate" ]]; then + echo "==> mode ${mode} run ${mode_run_index}: using same-capture signal conditioning before measured probe" + fi + + if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" == "separate" ]]; then + mkdir -p "${readiness_dir}" + echo "==> mode ${mode} run ${mode_run_index}: proving Tethys signal readiness before measured probe" + readiness_pass=1 + readiness_status=0 + readiness_artifact_dir="" + readiness_reason="no readiness attempts ran" + readiness_attempt_jsons=() + for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do + attempt_dir="${readiness_dir}/attempt-${readiness_attempt}" + readiness_log="${attempt_dir}/signal-readiness-run.log" + readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json" + mkdir -p "${attempt_dir}" + echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events" + run_mode_probe \ + "${width}" \ + "${height}" \ + "${fps}" \ + "${audio_delay_us}" \ + "${video_delay_us}" \ + "${attempt_dir}" \ + "${readiness_log}" \ + "${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \ + "${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \ + "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ + 0 + readiness_status=${RUN_MODE_PROBE_STATUS} + readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")" + set +e + readiness_reason="$( + write_signal_readiness_attempt_result \ + "${readiness_attempt}" \ + "${readiness_artifact_dir}" \ + "${readiness_status}" \ + "${readiness_log}" \ + "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ + "${readiness_attempt_json}" 2>&1 + )" + readiness_pass=$? + set -e + readiness_attempt_jsons+=("${readiness_attempt_json}") + if [[ "${readiness_pass}" -eq 0 ]]; then + echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}" + break + fi + [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed" + echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}" + if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then + echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness" + sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}" + fi + done + write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}" + if [[ "${readiness_pass}" -ne 0 ]]; then + [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed" + echo " ↪ signal readiness failed: ${readiness_reason}" + write_signal_readiness_failure \ + "${mode}" \ + "${width}" \ + "${height}" \ + "${fps}" \ + "${video_delay_us}" \ + "${audio_delay_us}" \ "${readiness_status}" \ "${readiness_log}" \ - "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ - "${readiness_attempt_json}" 2>&1 - )" - readiness_pass=$? - set -e - readiness_attempt_jsons+=("${readiness_attempt_json}") - if [[ "${readiness_pass}" -eq 0 ]]; then - echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}" - break + "${readiness_artifact_dir}" \ + "${readiness_attempts_json}" \ + "${readiness_reason}" \ + "${mode_result}" + annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "readiness-failure" + cp "${mode_result}" "${seed_result}" + if [[ "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then + break 2 + fi + continue fi - [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed" - echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}" - if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then - echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness" - sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}" - fi - done - write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}" - if [[ "${readiness_pass}" -ne 0 ]]; then - [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed" - echo " ↪ signal readiness failed: ${readiness_reason}" - write_signal_readiness_failure \ - "${mode}" \ - "${width}" \ - "${height}" \ - "${fps}" \ - "${video_delay_us}" \ - "${audio_delay_us}" \ - "${readiness_status}" \ - "${readiness_log}" \ - "${readiness_artifact_dir}" \ - "${readiness_attempts_json}" \ - "${readiness_reason}" \ - "${mode_result}" - cp "${mode_result}" "${seed_result}" - if [[ "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then - break - fi - continue + echo " ↪ signal readiness passed: artifact_dir=${readiness_artifact_dir}" fi - echo " ↪ signal readiness passed: artifact_dir=${readiness_artifact_dir}" - fi - run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}" - run_status=${RUN_MODE_PROBE_STATUS} - artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")" - write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}" - cp "${mode_result}" "${seed_result}" + echo "==> mode ${mode} run ${mode_run_index}: running seed probe" + run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}" + run_status=${RUN_MODE_PROBE_STATUS} + artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")" + write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}" + annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "seed" + cp "${mode_result}" "${seed_result}" - if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then - write_tune_candidate_env "${seed_result}" "${tune_env}" - # shellcheck disable=SC1090 - source "${tune_env}" - if [[ "${tune_ready:-false}" == "true" ]]; then - echo "==> mode ${mode}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}" - echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms" - run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}" - tuned_status=${RUN_MODE_PROBE_STATUS} - tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")" - write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}" - cp "${tuned_result}" "${mode_result}" - run_status=${tuned_status} - else - echo " ↪ tune skipped: ${tune_reason:-not ready}" + if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then + write_tune_candidate_env "${seed_result}" "${tune_env}" + # shellcheck disable=SC1090 + source "${tune_env}" + if [[ "${tune_ready:-false}" == "true" ]]; then + echo "==> mode ${mode} run ${mode_run_index}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}" + echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms" + run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}" + tuned_status=${RUN_MODE_PROBE_STATUS} + tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")" + write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}" + annotate_mode_result "${tuned_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "tuned" + cp "${tuned_result}" "${mode_result}" + run_status=${tuned_status} + else + echo " ↪ tune skipped: ${tune_reason:-not ready}" + fi fi - fi - if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then - break - fi + if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then + break 2 + fi + done done summarize_matrix @@ -1999,6 +2410,11 @@ echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}" echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}" echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}" echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}" +echo "mode_static_calibration_json: ${MATRIX_STATIC_JSON}" +echo "mode_static_calibration_csv: ${MATRIX_STATIC_CSV}" +echo "mode_static_calibration_txt: ${MATRIX_STATIC_TXT}" +echo "mode_static_calibration_env: ${MATRIX_STATIC_ENV}" +echo "mode_matrix_run_log: ${MATRIX_RUN_LOG}" if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}" import json diff --git a/server/Cargo.toml b/server/Cargo.toml index 196ecf5..0a35a4f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.27" +version = "0.19.28" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index 807b054..d61d85e 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -471,9 +471,13 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"", "output_delay_calibration", "write_tune_candidate_env", + "annotate_mode_result", + "LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1}", + "mode-static-calibration.json", + "mode-matrix-run.log", "mode-result-seed.json", "mode-result-tuned.json", - "==> mode ${mode}: confirming tuned delays", + "==> mode ${mode} run ${mode_run_index}: confirming tuned delays", "calibration_ready", "calibration_video_target_offset_us", "calibration_audio_target_offset_us", @@ -484,25 +488,26 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "signature_coverage", "paired coded signatures", "signature_missing_codes", - "REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"", - "REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"", - "REMOTE_CAPTURE_STACK=\"${REMOTE_CAPTURE_STACK}\"", - "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=\"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"", - "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=\"${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"", - "REMOTE_CAPTURE_READY_SETTLE_SECONDS=\"${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"", - "PROBE_PREBUILD=0", - "VIDEO_SIZE=\"${width}x${height}\"", - "VIDEO_FPS=\"${fps}\"", - "REMOTE_EXPECT_UVC_WIDTH=\"${width}\"", - "REMOTE_EXPECT_UVC_HEIGHT=\"${height}\"", - "REMOTE_EXPECT_UVC_FPS=\"${fps}\"", - "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${audio_delay_us}\"", - "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${video_delay_us}\"", - "LESAVKA_OUTPUT_DELAY_APPLY=0", - "LESAVKA_OUTPUT_DELAY_SAVE=0", - "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"", - "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"", - "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=\"${min_pairs}\"", + "probe_env=(", + "\"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}\"", + "\"REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}\"", + "\"REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}\"", + "\"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"", + "\"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"", + "\"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"", + "\"PROBE_PREBUILD=0\"", + "\"VIDEO_SIZE=${width}x${height}\"", + "\"VIDEO_FPS=${fps}\"", + "\"REMOTE_EXPECT_UVC_WIDTH=${width}\"", + "\"REMOTE_EXPECT_UVC_HEIGHT=${height}\"", + "\"REMOTE_EXPECT_UVC_FPS=${fps}\"", + "\"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}\"", + "\"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}\"", + "\"LESAVKA_OUTPUT_DELAY_APPLY=0\"", + "\"LESAVKA_OUTPUT_DELAY_SAVE=0\"", + "\"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"", + "\"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"", + "\"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}\"", "sync did not pass", "freshness did not pass", "video hiccups",