#!/usr/bin/env bash # scripts/manual/run_server_to_rc_mode_matrix.sh # Manual: validate server-generated UVC/UAC output against the RC target across # the UVC modes the UI advertises. This is still a hardware-in-the-loop probe: # it captures the real Tethys UVC/UAC endpoints and summarizes sync, # freshness, and smoothness for each mode. # Not part of CI: it needs the Theia/Tethys lab hosts and live USB gadget state. # # Reconfigure mode is intentionally a fast runtime path: it updates the remote # Lesavka env files and cycles the UVC gadget, but it does not rebuild or # reinstall the server binary for each mode. set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" if (( EUID == 0 )); then printf 'Do not run this matrix with local sudo.\n' >&2 printf 'Run it as your normal workstation user; the script will request sudo on the remote server host only when reconfiguring UVC.\n' >&2 exit 64 fi TETHYS_HOST=${TETHYS_HOST:-tethys} LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia} LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112} LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto} LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https} LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server} LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto} SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"} LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30} LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}} LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}} LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-135090} LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}} LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=162659,1280x720@30=135090,1920x1080@20=160045,1920x1080@30=127952} LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080} LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30} LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera} LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture} LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured} LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0} LESAVKA_SERVER_RC_RECONFIGURE_REF=${LESAVKA_SERVER_RC_RECONFIGURE_REF:-master} LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0} LESAVKA_SERVER_RC_RECONFIGURE_COMMAND=${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND:-} LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime} LESAVKA_SERVER_RC_RECONFIGURE_CODEC=${LESAVKA_SERVER_RC_RECONFIGURE_CODEC:-mjpeg} LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1} LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD=${LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD:-1} LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS=${LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS:-4} LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0} LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1} LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD=${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD:-} LESAVKA_SERVER_RC_START_DELAY_SECONDS=${LESAVKA_SERVER_RC_START_DELAY_SECONDS:-0} LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1} LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60} LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6} LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3} LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1} LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1} LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1} LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1} LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-13} LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000} LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80} LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000} LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000} LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1} LESAVKA_SERVER_RC_VERBOSE_PROBES=${LESAVKA_SERVER_RC_VERBOSE_PROBES:-1} LESAVKA_SERVER_RC_STATIC_MIN_RUNS=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS:-3} LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US:-30000} LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS:-35} LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS:-20} LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS:-1} LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS:-0} LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100} LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250} LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS:-${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}} LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS:-1} LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS=${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS:-1} LESAVKA_SERVER_RC_MIN_CODED_PAIRS=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS:-${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}} LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS:-0} LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0} LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1} LESAVKA_SERVER_RC_SIGNAL_READY_MODE=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE:-conditioned_capture} LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3} LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12} LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS:-1} LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4} LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5} LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS:-12} LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS:-1} LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS:-1} LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES:-${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}} LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW:-auto} LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0} LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0} LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS:-1} LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS:-1} LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12} LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES:-12} LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES:-5} LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS:-2} PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20} PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4} PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0} REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1} LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0} STAMP="$(date +%Y%m%d-%H%M%S)" LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp} MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc-mode-matrix-${STAMP}"} MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json" MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv" MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt" MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json" MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env" MATRIX_STATIC_JSON="${MATRIX_REPORT_DIR}/mode-static-calibration.json" MATRIX_STATIC_CSV="${MATRIX_REPORT_DIR}/mode-static-calibration.csv" MATRIX_STATIC_TXT="${MATRIX_REPORT_DIR}/mode-static-calibration.txt" MATRIX_STATIC_ENV="${MATRIX_REPORT_DIR}/mode-static-calibration.env" MATRIX_RUN_LOG="${MATRIX_REPORT_DIR}/mode-matrix-run.log" mkdir -p "${MATRIX_REPORT_DIR}" exec > >(tee -a "${MATRIX_RUN_LOG}") 2>&1 mode_id() { local mode=$1 printf '%s\n' "${mode//@/_}" | tr -c '[:alnum:]_.-' '_' } parse_mode() { local mode=$1 if [[ ! "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' "${mode}" >&2 exit 64 fi printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" } lookup_mode_delay_us() { local mode=$1 local delay_map=$2 local default_value=$3 local entry key value IFS=',' read -r -a delay_entries <<<"${delay_map}" for entry in "${delay_entries[@]}"; do key=${entry%%=*} value=${entry#*=} if [[ "${key}" == "${mode}" && "${value}" =~ ^-?[0-9]+$ ]]; then printf '%s\n' "${value}" return 0 fi done printf '%s\n' "${default_value}" } lookup_audio_delay_us() { lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}" } lookup_video_delay_us() { lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}" } artifact_dir_from_log() { local log_path=$1 local search_dir=$2 local artifact_dir artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${log_path}" 2>/dev/null | tail -n1 || true)" if [[ -z "${artifact_dir}" ]]; then artifact_dir="$(find "${search_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)" fi printf '%s\n' "${artifact_dir}" } RUN_MODE_PROBE_STATUS=0 run_mode_probe() { local width=$1 local height=$2 local fps=$3 local audio_delay_us=$4 local video_delay_us=$5 local output_dir=$6 local log_path=$7 local probe_duration_seconds=${8:-${PROBE_DURATION_SECONDS}} local probe_warmup_seconds=${9:-${PROBE_WARMUP_SECONDS}} local min_pairs=${10:-${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}} local conditioning_seconds=${11:-${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS}} local analysis_timeline_window=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW} if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" == "0" ]]; then conditioning_seconds=0 fi if [[ "${analysis_timeline_window}" == "auto" ]]; then if [[ "${conditioning_seconds}" =~ ^[0-9]+$ && "${conditioning_seconds}" -gt 0 ]]; then analysis_timeline_window=1 else analysis_timeline_window=0 fi fi local -a probe_env=( "TETHYS_HOST=${TETHYS_HOST}" "LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST}" "LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST}" "LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR}" "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME}" "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN}" "SSH_OPTS=${SSH_OPTS}" "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}" "REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}" "REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}" "REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE}" "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}" "PROBE_PREBUILD=0" "VIDEO_SIZE=${width}x${height}" "VIDEO_FPS=${fps}" "VIDEO_FORMAT=mjpeg" "REMOTE_EXPECT_UVC_WIDTH=${width}" "REMOTE_EXPECT_UVC_HEIGHT=${height}" "REMOTE_EXPECT_UVC_FPS=${fps}" "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}" "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}" "LESAVKA_OUTPUT_DELAY_APPLY=0" "LESAVKA_OUTPUT_DELAY_SAVE=0" "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0" "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${min_pairs}" "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}" "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES}" "PROBE_CONDITIONING_SECONDS=${conditioning_seconds}" "PROBE_CONDITIONING_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}" "PROBE_CONDITIONING_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}" "PROBE_CONDITIONING_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}" "PROBE_DURATION_SECONDS=${probe_duration_seconds}" "PROBE_WARMUP_SECONDS=${probe_warmup_seconds}" "ANALYSIS_TIMELINE_WINDOW=${analysis_timeline_window}" "LOCAL_OUTPUT_DIR=${output_dir}" ) set +e if [[ "${LESAVKA_SERVER_RC_VERBOSE_PROBES}" == "0" ]]; then env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" >"${log_path}" 2>&1 RUN_MODE_PROBE_STATUS=$? else env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}" RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]} fi set -e } write_tune_candidate_env() { local mode_result=$1 local output_env=$2 python3 - <<'PY' \ "${mode_result}" \ "${output_env}" \ "${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \ "${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \ "${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \ "${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \ "${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" import json import math import pathlib import shlex import sys ( result_path, output_env_path, min_pairs_raw, max_abs_skew_raw, max_drift_raw, max_step_raw, min_change_raw, ) = sys.argv[1:8] def env_line(key, value): return f"{key}={shlex.quote(str(value))}\n" def as_int(value, default=0): try: return int(str(value).strip()) except Exception: return default def as_float(value, default=0.0): try: result = float(str(value).strip()) except Exception: return default return result if math.isfinite(result) else default result = json.loads(pathlib.Path(result_path).read_text()) calibration = result.get("output_delay_calibration") or {} sync = result.get("sync") or {} min_pairs = max(1, as_int(min_pairs_raw, 13)) max_abs_skew_ms = max(1.0, as_float(max_abs_skew_raw, 1000.0)) max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0)) max_step_us = max(1, as_int(max_step_raw, 500_000)) min_change_us = max(0, as_int(min_change_raw, 5_000)) current_audio = as_int(result.get("audio_delay_us"), 0) current_video = as_int(result.get("video_delay_us"), 0) target_audio = as_int(calibration.get("audio_target_offset_us"), current_audio) target_video = as_int(calibration.get("video_target_offset_us"), current_video) paired = as_int(calibration.get("paired_event_count"), as_int(sync.get("paired_event_count"), 0)) drift_ms = as_float(calibration.get("drift_ms"), as_float(sync.get("drift_ms"), 0.0)) max_abs_skew_ms_observed = as_float( calibration.get("max_abs_skew_ms"), as_float(sync.get("p95_abs_skew_ms"), 0.0), ) delta_audio = target_audio - current_audio delta_video = target_video - current_video raw_delta_us = as_int( calibration.get("raw_device_delta_us"), delta_video if abs(delta_video) >= abs(delta_audio) else -delta_audio, ) reasons = [] if result.get("run_status") != 0: reasons.append(f"seed probe exited {result.get('run_status')}") if paired < min_pairs: reasons.append(f"paired_event_count {paired} < {min_pairs}") if max_abs_skew_ms_observed > max_abs_skew_ms: reasons.append( f"max_abs_skew_ms {max_abs_skew_ms_observed:.1f} > {max_abs_skew_ms:.1f}" ) if abs(drift_ms) > max_drift_ms: reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}") if abs(raw_delta_us) > max_step_us: reasons.append(f"raw delay correction {raw_delta_us:+d}us exceeds {max_step_us}us") if target_audio < 0 or target_video < 0: reasons.append( f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}" ) if abs(delta_audio) < min_change_us and abs(delta_video) < min_change_us: reasons.append( f"target change audio={delta_audio:+d}us video={delta_video:+d}us is below {min_change_us}us" ) ready = not reasons values = { "tune_ready": str(ready).lower(), "tune_reason": "ready" if ready else "; ".join(reasons), "tune_audio_delay_us": target_audio, "tune_video_delay_us": target_video, "tune_audio_delta_us": delta_audio, "tune_video_delta_us": delta_video, "tune_paired_event_count": paired, "tune_drift_ms": f"{drift_ms:.3f}", "tune_max_abs_skew_ms": f"{max_abs_skew_ms_observed:.3f}", "tune_raw_delta_us": raw_delta_us, } with pathlib.Path(output_env_path).open("w") as handle: for key, value in values.items(): handle.write(env_line(key, value)) PY } signal_readiness_passed() { local artifact_dir=$1 local min_pairs=$2 python3 - <<'PY' "${artifact_dir}" "${min_pairs}" import json import pathlib import sys artifact_dir = pathlib.Path(sys.argv[1]) min_pairs = int(sys.argv[2]) def load_json(path): try: return json.loads(path.read_text()) except Exception: return {} report = load_json(artifact_dir / "report.json") paired = int(report.get("paired_event_count") or 0) video = int(report.get("video_event_count") or 0) audio = int(report.get("audio_event_count") or 0) verdict = report.get("verdict") or {} passed = paired >= min_pairs and video >= min_pairs and audio >= min_pairs and verdict.get("passed") is True if not passed: reason = ( f"signal readiness needs at least {min_pairs} paired coded events with sync pass; " f"saw paired={paired}, video={video}, audio={audio}, sync={verdict.get('status', 'unknown')}" ) print(reason) raise SystemExit(0 if passed else 1) PY } write_signal_readiness_attempt_result() { local attempt=$1 local artifact_dir=$2 local run_status=$3 local run_log=$4 local min_pairs=$5 local output_json=$6 python3 - <<'PY' \ "${attempt}" \ "${artifact_dir}" \ "${run_status}" \ "${run_log}" \ "${min_pairs}" \ "${output_json}" import json import pathlib import re import sys ( attempt_raw, artifact_dir_raw, run_status_raw, run_log_raw, min_pairs_raw, output_json_raw, ) = sys.argv[1:] def as_int(value, default=0): try: return int(str(value).strip()) except Exception: return default def as_float(value, default=0.0): try: return float(str(value).strip()) except Exception: return default def load_json(path): try: return json.loads(path.read_text()) except Exception: return {} def read_text(path): try: return path.read_text(errors="replace") except Exception: return "" attempt = as_int(attempt_raw) run_status = as_int(run_status_raw) min_pairs = as_int(min_pairs_raw, 1) run_log = pathlib.Path(run_log_raw) artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else None if artifact_dir is None or not artifact_dir.exists(): match = re.findall(r"^artifact_dir:\s*(.+)$", read_text(run_log), flags=re.MULTILINE) if match: artifact_dir = pathlib.Path(match[-1].strip()) if artifact_dir is None: artifact_dir = pathlib.Path("__missing_signal_readiness_artifact__") report = load_json(artifact_dir / "report.json") timeline = load_json(artifact_dir / "server-output-timeline.json") error_log = artifact_dir / "analysis-error.log" error_text = read_text(error_log) if not error_text: error_log = artifact_dir / "analysis-error.log.tmp" error_text = read_text(error_log) if not error_text: error_text = read_text(run_log) events = timeline.get("events") or [] server_event_count = len(events) server_video_handoff_count = sum( 1 for event in events if event.get("video_feed_unix_ns") is not None or event.get("video_feed_monotonic_us") is not None ) server_audio_handoff_count = sum( 1 for event in events if event.get("audio_push_unix_ns") is not None or event.get("audio_push_monotonic_us") is not None ) verdict = report.get("verdict") or {} signature_coverage = report.get("signature_coverage") or {} video_events = as_int(report.get("video_event_count")) audio_events = as_int(report.get("audio_event_count")) paired_events = as_int(report.get("paired_event_count")) coded_pairs = as_int(signature_coverage.get("paired_event_count"), paired_events) expected_pairs = as_int(signature_coverage.get("expected_event_count")) raw_activity_delta_ms = as_float(report.get("activity_start_delta_ms")) analyzer_failure = "" failure_match = re.search( r"coded pulse common window removed one stream entirely;[^\n]*" r"\(video=(?P