diff --git a/Cargo.lock b/Cargo.lock index 482ec1d..1a71efc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.16" +version = "0.19.17" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.16" +version = "0.19.17" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.16" +version = "0.19.17" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 31b7d46..4cefc9e 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.16" +version = "0.19.17" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 8d5beb0..41c24e1 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.16" +version = "0.19.17" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh index eb4dd97..c3b9f60 100755 --- a/scripts/manual/run_server_to_rc_mode_matrix.sh +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -58,6 +58,11 @@ LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECOND LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3} LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1} LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1} +LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1} +LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1} +LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-3} +LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80} +LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000} LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100} @@ -90,6 +95,8 @@ MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc- MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json" MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv" MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt" +MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json" +MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env" mkdir -p "${MATRIX_REPORT_DIR}" mode_id() { @@ -131,6 +138,152 @@ lookup_video_delay_us() { lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}" } +artifact_dir_from_log() { + local log_path=$1 + local search_dir=$2 + local artifact_dir + artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${log_path}" 2>/dev/null | tail -n1 || true)" + if [[ -z "${artifact_dir}" ]]; then + artifact_dir="$(find "${search_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)" + fi + printf '%s\n' "${artifact_dir}" +} + +RUN_MODE_PROBE_STATUS=0 +run_mode_probe() { + local width=$1 + local height=$2 + local fps=$3 + local audio_delay_us=$4 + local video_delay_us=$5 + local output_dir=$6 + local log_path=$7 + + set +e + TETHYS_HOST="${TETHYS_HOST}" \ + LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \ + LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \ + LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \ + LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \ + LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \ + SSH_OPTS="${SSH_OPTS}" \ + REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \ + REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \ + REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \ + REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \ + REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ + REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \ + PROBE_PREBUILD=0 \ + VIDEO_SIZE="${width}x${height}" \ + VIDEO_FPS="${fps}" \ + VIDEO_FORMAT=mjpeg \ + REMOTE_EXPECT_UVC_WIDTH="${width}" \ + REMOTE_EXPECT_UVC_HEIGHT="${height}" \ + REMOTE_EXPECT_UVC_FPS="${fps}" \ + LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${audio_delay_us}" \ + LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \ + LESAVKA_OUTPUT_DELAY_APPLY=0 \ + LESAVKA_OUTPUT_DELAY_SAVE=0 \ + LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \ + LESAVKA_OUTPUT_DELAY_MIN_PAIRS="${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \ + LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ + LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \ + LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \ + LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \ + PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \ + PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \ + PROBE_WARMUP_SECONDS="${PROBE_WARMUP_SECONDS}" \ + LOCAL_OUTPUT_DIR="${output_dir}" \ + "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}" + RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]} + set -e +} + +write_tune_candidate_env() { + local mode_result=$1 + local output_env=$2 + python3 - <<'PY' \ + "${mode_result}" \ + "${output_env}" \ + "${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \ + "${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ + "${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" +import json +import math +import pathlib +import shlex +import sys + +result_path, output_env_path, min_pairs_raw, max_drift_raw, min_change_raw = sys.argv[1:6] + + +def env_line(key, value): + return f"{key}={shlex.quote(str(value))}\n" + + +def as_int(value, default=0): + try: + return int(str(value).strip()) + except Exception: + return default + + +def as_float(value, default=0.0): + try: + result = float(str(value).strip()) + except Exception: + return default + return result if math.isfinite(result) else default + + +result = json.loads(pathlib.Path(result_path).read_text()) +calibration = result.get("output_delay_calibration") or {} +sync = result.get("sync") or {} +min_pairs = max(1, as_int(min_pairs_raw, 3)) +max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0)) +min_change_us = max(0, as_int(min_change_raw, 5_000)) +current_audio = as_int(result.get("audio_delay_us"), 0) +current_video = as_int(result.get("video_delay_us"), 0) +target_audio = as_int(calibration.get("audio_target_offset_us"), current_audio) +target_video = as_int(calibration.get("video_target_offset_us"), current_video) +paired = as_int(calibration.get("paired_event_count"), as_int(sync.get("paired_event_count"), 0)) +drift_ms = as_float(calibration.get("drift_ms"), as_float(sync.get("drift_ms"), 0.0)) +delta_audio = target_audio - current_audio +delta_video = target_video - current_video +reasons = [] + +if result.get("run_status") != 0: + reasons.append(f"seed probe exited {result.get('run_status')}") +if paired < min_pairs: + reasons.append(f"paired_event_count {paired} < {min_pairs}") +if abs(drift_ms) > max_drift_ms: + reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}") +if target_audio < 0 or target_video < 0: + reasons.append( + f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}" + ) +if abs(delta_audio) < min_change_us and abs(delta_video) < min_change_us: + reasons.append( + f"target change audio={delta_audio:+d}us video={delta_video:+d}us is below {min_change_us}us" + ) + +ready = not reasons +values = { + "tune_ready": str(ready).lower(), + "tune_reason": "ready" if ready else "; ".join(reasons), + "tune_audio_delay_us": target_audio, + "tune_video_delay_us": target_video, + "tune_audio_delta_us": delta_audio, + "tune_video_delta_us": delta_video, + "tune_paired_event_count": paired, + "tune_drift_ms": f"{drift_ms:.3f}", +} +with pathlib.Path(output_env_path).open("w") as handle: + for key, value in values.items(): + handle.write(env_line(key, value)) +PY +} + discover_local_webcam_modes() { if ! command -v python3 >/dev/null 2>&1; then printf 'LESAVKA_SERVER_RC_MODES=auto requires python3 for local webcam mode discovery.\n' >&2 @@ -828,6 +981,8 @@ artifact = { "p95_abs_skew_ms": as_float(calibration.get("p95_abs_skew_ms"), 0.0), "max_abs_skew_ms": as_float(calibration.get("max_abs_skew_ms"), 0.0), "drift_ms": as_float(calibration.get("drift_ms"), 0.0), + "active_audio_offset_us": as_int(calibration.get("active_audio_offset_us"), as_int(audio_delay_raw)), + "active_video_offset_us": as_int(calibration.get("active_video_offset_us"), as_int(video_delay_raw)), "audio_offset_adjust_us": as_int(calibration.get("audio_offset_adjust_us"), 0), "video_offset_adjust_us": as_int(calibration.get("video_offset_adjust_us"), 0), "audio_target_offset_us": as_int(calibration.get("audio_target_offset_us"), 0), @@ -840,35 +995,127 @@ PY } summarize_matrix() { - python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" + python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" import csv import json import pathlib +import shlex import sys root = pathlib.Path(sys.argv[1]) summary_json = pathlib.Path(sys.argv[2]) summary_csv = pathlib.Path(sys.argv[3]) summary_txt = pathlib.Path(sys.argv[4]) +delay_json = pathlib.Path(sys.argv[5]) +delay_env = pathlib.Path(sys.argv[6]) results = [] for path in sorted(root.glob("*/mode-result.json")): try: - results.append(json.loads(path.read_text())) + result = json.loads(path.read_text()) + seed_path = path.with_name("mode-result-seed.json") + tuned_path = path.with_name("mode-result-tuned.json") + if seed_path.exists(): + result["seed_result_json"] = str(seed_path) + try: + seed = json.loads(seed_path.read_text()) + result["seed_audio_delay_us"] = seed.get("audio_delay_us") + result["seed_video_delay_us"] = seed.get("video_delay_us") + except Exception: + pass + if tuned_path.exists(): + result["tuned_result_json"] = str(tuned_path) + results.append(result) except Exception: continue +delay_recommendations = {} +video_delay_entries = [] +audio_delay_entries = [] +for result in results: + mode = result.get("mode") + if not mode: + continue + sync = result.get("sync") or {} + calibration = result.get("output_delay_calibration") or {} + confirmed = sync.get("passed") is True + candidate_video = calibration.get("video_target_offset_us") + candidate_audio = calibration.get("audio_target_offset_us") + video_delay = result.get("video_delay_us") + audio_delay = result.get("audio_delay_us") + status = "confirmed" if confirmed else "tested" + candidate_available = ( + calibration.get("decision") in {"ready", "refused"} + and (calibration.get("paired_event_count") or 0) > 0 + and isinstance(candidate_video, int) + and isinstance(candidate_audio, int) + ) + if not confirmed and candidate_available: + video_delay = candidate_video + audio_delay = candidate_audio + status = "candidate_unconfirmed" + elif not confirmed: + status = "unavailable" + video_delay = None + audio_delay = None + if status != "unavailable" and isinstance(video_delay, int) and isinstance(audio_delay, int): + video_delay_entries.append(f"{mode}={video_delay}") + audio_delay_entries.append(f"{mode}={audio_delay}") + delay_recommendations[mode] = { + "status": status, + "audio_delay_us": audio_delay, + "video_delay_us": video_delay, + "tested_audio_delay_us": result.get("audio_delay_us"), + "tested_video_delay_us": result.get("video_delay_us"), + "sync_status": sync.get("status"), + "median_skew_ms": sync.get("median_skew_ms"), + "paired_event_count": sync.get("paired_event_count"), + } + summary = { "schema": "lesavka.server-rc-mode-matrix-summary.v1", "artifact_dir": str(root), "passed": bool(results) and all(result.get("passed") for result in results), "mode_count": len(results), + "delay_recommendations_json": str(delay_json), + "delay_recommendations_env": str(delay_env), + "delay_recommendations": delay_recommendations, "results": results, } summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") +delay_json.write_text( + json.dumps( + { + "schema": "lesavka.server-rc-mode-delay-recommendations.v1", + "artifact_dir": str(root), + "video_delays_us": { + mode: entry.get("video_delay_us") + for mode, entry in delay_recommendations.items() + }, + "audio_delays_us": { + mode: entry.get("audio_delay_us") + for mode, entry in delay_recommendations.items() + }, + "recommendations": delay_recommendations, + }, + indent=2, + sort_keys=True, + ) + + "\n" +) +delay_env.write_text( + "LESAVKA_SERVER_RC_MODE_DELAYS_US=" + + shlex.quote(",".join(video_delay_entries)) + + "\n" + + "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=" + + shlex.quote(",".join(audio_delay_entries)) + + "\n" +) fieldnames = [ "mode", "passed", + "seed_video_delay_us", + "seed_audio_delay_us", "video_delay_us", "audio_delay_us", "sync_status", @@ -898,6 +1145,8 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle: writer.writerow({ "mode": result.get("mode"), "passed": result.get("passed"), + "seed_video_delay_us": result.get("seed_video_delay_us"), + "seed_audio_delay_us": result.get("seed_audio_delay_us"), "video_delay_us": result.get("video_delay_us"), "audio_delay_us": result.get("audio_delay_us"), "sync_status": (result.get("sync") or {}).get("status"), @@ -946,8 +1195,17 @@ for result in results: f"video_target={calibration.get('video_target_offset_us', 0)}us " f"audio_target={calibration.get('audio_target_offset_us', 0)}us" ) + if "seed_video_delay_us" in result or "seed_audio_delay_us" in result: + lines.append( + " tuning: " + f"seed video={result.get('seed_video_delay_us', 0)}us audio={result.get('seed_audio_delay_us', 0)}us -> " + f"tested video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us" + ) for reason in result.get("failure_reasons") or []: lines.append(f" reason: {reason}") +if video_delay_entries or audio_delay_entries: + lines.append(f"- recommended video delays: {','.join(video_delay_entries)}") + lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}") summary_txt.write_text("\n".join(lines) + "\n") print("\n".join(lines)) PY @@ -965,6 +1223,7 @@ echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}" echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}" +echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}" echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s" @@ -984,54 +1243,39 @@ for mode in "${modes[@]}"; do mode_dir="${MATRIX_REPORT_DIR}/${id}" mode_log="${mode_dir}/mode-run.log" mode_result="${mode_dir}/mode-result.json" + seed_result="${mode_dir}/mode-result-seed.json" + tuned_log="${mode_dir}/mode-tuned-run.log" + tuned_result="${mode_dir}/mode-result-tuned.json" + tune_env="${mode_dir}/mode-tune-candidate.env" mkdir -p "${mode_dir}" echo "==> mode ${mode}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}" reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}" - set +e - TETHYS_HOST="${TETHYS_HOST}" \ - LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \ - LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \ - LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \ - LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \ - LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \ - SSH_OPTS="${SSH_OPTS}" \ - REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \ - REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \ - REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \ - REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \ - REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ - REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \ - PROBE_PREBUILD=0 \ - VIDEO_SIZE="${width}x${height}" \ - VIDEO_FPS="${fps}" \ - VIDEO_FORMAT=mjpeg \ - REMOTE_EXPECT_UVC_WIDTH="${width}" \ - REMOTE_EXPECT_UVC_HEIGHT="${height}" \ - REMOTE_EXPECT_UVC_FPS="${fps}" \ - LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${audio_delay_us}" \ - LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \ - LESAVKA_OUTPUT_DELAY_APPLY=0 \ - LESAVKA_OUTPUT_DELAY_SAVE=0 \ - LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \ - LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \ - LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \ - LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \ - PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \ - PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \ - PROBE_WARMUP_SECONDS="${PROBE_WARMUP_SECONDS}" \ - LOCAL_OUTPUT_DIR="${mode_dir}" \ - "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${mode_log}" - run_status=${PIPESTATUS[0]} - set -e - - artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${mode_log}" | tail -n1)" - if [[ -z "${artifact_dir}" ]]; then - artifact_dir="$(find "${mode_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)" - fi + run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}" + run_status=${RUN_MODE_PROBE_STATUS} + artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")" write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}" + cp "${mode_result}" "${seed_result}" + + if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then + write_tune_candidate_env "${seed_result}" "${tune_env}" + # shellcheck disable=SC1090 + source "${tune_env}" + if [[ "${tune_ready:-false}" == "true" ]]; then + echo "==> mode ${mode}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}" + echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms" + run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}" + tuned_status=${RUN_MODE_PROBE_STATUS} + tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")" + write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}" + cp "${tuned_result}" "${mode_result}" + run_status=${tuned_status} + else + echo " ↪ tune skipped: ${tune_reason:-not ready}" + fi + fi if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then break @@ -1045,6 +1289,8 @@ echo "artifact_dir: ${MATRIX_REPORT_DIR}" echo "mode_matrix_summary_json: ${MATRIX_SUMMARY_JSON}" echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}" echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}" +echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}" +echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}" if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}" import json diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index 496178f..df656d0 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -675,7 +675,9 @@ write_output_delay_calibration() { "${LESAVKA_OUTPUT_DELAY_MAX_STEP_US}" \ "${LESAVKA_OUTPUT_DELAY_APPLY}" \ "${LESAVKA_OUTPUT_DELAY_APPLY_MODE}" \ - "${LESAVKA_OUTPUT_DELAY_SAVE}" + "${LESAVKA_OUTPUT_DELAY_SAVE}" \ + "${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}" \ + "${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}" import json import math import pathlib @@ -696,6 +698,8 @@ import sys apply_raw, apply_mode_raw, save_raw, + active_audio_raw, + active_video_raw, ) = sys.argv[1:] @@ -732,6 +736,8 @@ max_abs_skew_ms = max(1.0, as_float(max_abs_skew_raw, 5000.0)) max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0)) gain = min(max(as_float(gain_raw, 1.0), 0.01), 1.0) max_step_us = max(1, as_int(max_step_raw, 1_500_000)) +active_audio_offset_us = as_int(active_audio_raw, 0) +active_video_offset_us = as_int(active_video_raw, 0) paired = as_int(report.get("paired_event_count"), 0) median_skew_ms = as_float(report.get("median_skew_ms"), 0.0) @@ -771,12 +777,17 @@ if max_abs_observed_ms > max_abs_skew_ms: if abs(drift_ms) > max_drift_ms: refusal_reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}") +audio_target_offset_us = active_audio_offset_us + audio_delta_us +video_target_offset_us = active_video_offset_us + video_delta_us ready = not refusal_reasons decision = "ready" if ready else "refused" note = ( "direct UVC/UAC output-delay calibration: " f"median device skew {median_skew_ms:+.1f}ms, target={target}, " - f"audio {audio_delta_us:+d}us/video {video_delta_us:+d}us" + f"audio {active_audio_offset_us:+d}->{audio_target_offset_us:+d}us " + f"(delta {audio_delta_us:+d}us), " + f"video {active_video_offset_us:+d}->{video_target_offset_us:+d}us " + f"(delta {video_delta_us:+d}us)" ) if not ready: note = f"direct UVC/UAC output-delay calibration refused: {'; '.join(refusal_reasons)}" @@ -808,12 +819,14 @@ artifact = { "max_drift_ms": max_drift_ms, "gain": gain, "max_step_us": max_step_us, + "active_audio_offset_us": active_audio_offset_us, + "active_video_offset_us": active_video_offset_us, "raw_device_delta_us": raw_device_delta_us, "bounded_device_delta_us": bounded_delta_us, "audio_offset_adjust_us": audio_delta_us, "video_offset_adjust_us": video_delta_us, - "audio_target_offset_us": audio_delta_us, - "video_target_offset_us": video_delta_us, + "audio_target_offset_us": audio_target_offset_us, + "video_target_offset_us": video_target_offset_us, "refusal_reasons": refusal_reasons, "note": note, } @@ -825,8 +838,10 @@ env_values = { "output_delay_target": target, "output_delay_audio_delta_us": audio_delta_us, "output_delay_video_delta_us": video_delta_us, - "output_delay_audio_target_offset_us": audio_delta_us, - "output_delay_video_target_offset_us": video_delta_us, + "output_delay_active_audio_offset_us": active_audio_offset_us, + "output_delay_active_video_offset_us": active_video_offset_us, + "output_delay_audio_target_offset_us": audio_target_offset_us, + "output_delay_video_target_offset_us": video_target_offset_us, "output_delay_measured_skew_ms": f"{median_skew_ms:.3f}", "output_delay_paired_event_count": paired, "output_delay_drift_ms": f"{drift_ms:.3f}", @@ -1822,6 +1837,8 @@ maybe_apply_output_delay_calibration() { echo " ↪ output_delay_drift_ms=${output_delay_drift_ms:-0.0}" echo " ↪ output_delay_audio_delta_us=${output_delay_audio_delta_us:-0}" echo " ↪ output_delay_video_delta_us=${output_delay_video_delta_us:-0}" + echo " ↪ output_delay_active_audio_offset_us=${output_delay_active_audio_offset_us:-0}" + echo " ↪ output_delay_active_video_offset_us=${output_delay_active_video_offset_us:-0}" echo " ↪ output_delay_audio_target_offset_us=${output_delay_audio_target_offset_us:-0}" echo " ↪ output_delay_video_target_offset_us=${output_delay_video_target_offset_us:-0}" echo " ↪ output_delay_apply_mode=${output_delay_apply_mode:-${LESAVKA_OUTPUT_DELAY_APPLY_MODE}}" diff --git a/server/Cargo.toml b/server/Cargo.toml index 13c4b15..43214cd 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.16" +version = "0.19.17" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index 19cca79..b20d0c0 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -122,6 +122,12 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { "probe_media_origin\": \"server-generated\"", "probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"", "audio_after_video_positive", + "active_audio_offset_us", + "active_video_offset_us", + "audio_target_offset_us = active_audio_offset_us + audio_delta_us", + "video_target_offset_us = active_video_offset_us + video_delta_us", + "output_delay_active_audio_offset_us", + "output_delay_active_video_offset_us", "audio_target_offset_us", "video_target_offset_us", "output_delay_audio_target_offset_us", @@ -231,6 +237,10 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}", "LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}", "LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}", + "LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}", + "LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}", + "LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-3}", + "LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}", "Theia sudo password for %s", "==> priming remote sudo on ${LESAVKA_SERVER_HOST}", "==> prebuilding relay control/analyzer once for the mode matrix", @@ -257,9 +267,16 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { "mode-matrix-summary.json", "mode-matrix-summary.csv", "mode-matrix-summary.txt", + "mode-delay-recommendations.json", + "mode-delay-recommendations.env", "schema\": \"lesavka.server-rc-mode-result.v1\"", "schema\": \"lesavka.server-rc-mode-matrix-summary.v1\"", + "schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"", "output_delay_calibration", + "write_tune_candidate_env", + "mode-result-seed.json", + "mode-result-tuned.json", + "==> mode ${mode}: confirming tuned delays", "calibration_ready", "calibration_video_target_offset_us", "calibration_audio_target_offset_us",